{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Emotion Regression\n",
    "- Evaluation: Averaged Pearson correlations of all emotions\n",
    "- Model: BiLSTM with Self-attention\n",
    "- Word Embeddings: Random Init **vs** pretrained?\n",
    "- Training: direct training **vs** pretrained on Twitter sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "\n",
    "from tensorflow.contrib.rnn import LSTMStateTuple\n",
    "from scipy.stats import pearsonr\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bi-LSTM emotion regressor\n",
    "def create_cell(num_units, cell_type, forget_bias=1.0):\n",
    "    \"\"\"\n",
    "    cell: build a recurrent cell\n",
    "        num_units: number of hidden cell units\n",
    "        cell_type: LSTM, GRU, LN_LSTM\n",
    "    \"\"\"\n",
    "    if cell_type == \"LSTM\":\n",
    "        cell = tf.nn.rnn_cell.BasicLSTMCell(num_units, forget_bias=forget_bias)\n",
    "\n",
    "    elif cell_type == \"GRU\":\n",
    "        cell = tf.nn.rnn_cell.GRUCell(num_units)\n",
    "\n",
    "    elif cell_type == \"LN_LSTM\":\n",
    "        cell = tf.contrib.rnn.LayerNormBasicLSTMCell(\n",
    "            num_units,\n",
    "            forget_bias=forget_bias,\n",
    "            layer_norm=True)\n",
    "\n",
    "    else:\n",
    "        raise ValueError(\"Unknown cell type %s\" % cell_type)\n",
    "\n",
    "    return cell\n",
    "\n",
    "\n",
    "def build_rnn_cell(num_layers, num_units, cell_type, forget_bias=1.0):\n",
    "    \"\"\"\n",
    "    rnn_cell: build a multi-layer rnn cell\n",
    "        num_layers: number of hidden layers\n",
    "    \"\"\"\n",
    "    cell_seq = []\n",
    "    for i in range(num_layers):\n",
    "        cell = create_cell(num_units, cell_type, forget_bias)\n",
    "        cell_seq.append(cell)\n",
    "\n",
    "    if num_layers > 1:\n",
    "        rnn_cell = tf.nn.rnn_cell.MultiRNNCell(cell_seq)\n",
    "    else:\n",
    "        rnn_cell = cell_seq[0]\n",
    "\n",
    "    return rnn_cell\n",
    "\n",
    "\n",
    "def self_attention_scores(hs, num_attn_hidden):\n",
    "    \"\"\"\n",
    "    Pass hidden outputs to a feedforward net for self attention scores\n",
    "    Use 2-layer DNN with tanh as in https://arxiv.org/pdf/1804.06658.pdf\n",
    "        hs: [batch_size, max_time, num_hiddens]\n",
    "    Returns:\n",
    "        attn_scores: [batch_size, max_time, 1]\n",
    "    \"\"\"\n",
    "    attn_a1 = tf.nn.tanh(tf.layers.dense(hs, num_attn_hidden))\n",
    "    attn_a2 = tf.nn.tanh(tf.layers.dense(attn_a1, num_attn_hidden))\n",
    "    e = tf.layers.dense(attn_a2, 1)\n",
    "    attn_scores = tf.nn.softmax(tf.squeeze(e, axis=-1))\n",
    "    attn_scores = tf.expand_dims(attn_scores, -1)\n",
    "\n",
    "    return attn_scores\n",
    "\n",
    "\n",
    "def build_emotion_regressor(\n",
    "        embeddings, source_ids, num_layers, num_units, num_emotions,\n",
    "        cell_type, forget_bias=1.0, bidir=False, self_attention=False,\n",
    "        num_attn_hidden=128, dtype=tf.float32, name=\"encoder\"):\n",
    "    \"\"\"\n",
    "    Emotion Regressor for sentence following https://arxiv.org/abs/1804.06658\n",
    "    Christos Baziotis et al., arXiv (2018), NTUA-SLP at SemEval-2018\n",
    "        Task 1: Predicting Affective Content in Tweets\n",
    "        with Deep Attentive RNNs and Transfer Learning\n",
    "\n",
    "    encoder: build rnn encoder for Seq2seq\n",
    "        source_ids: [batch_size, max_time]\n",
    "        bidir: bidirectional or unidirectional\n",
    "\n",
    "    Returns:\n",
    "        encoder_outputs: [batch_size, max_time, num_units]\n",
    "        encoder_states: (StateTuple(shape=(batch_size, num_units)), ...)\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(name):\n",
    "        # embedding lookup, embed_inputs: [max_time, batch_size, num_units]\n",
    "        embed_inputs = tf.nn.embedding_lookup(embeddings, source_ids)\n",
    "        output_layer = tf.layers.Dense(num_emotions, name=\"output_layer\")\n",
    "\n",
    "        # bidirectional\n",
    "        if bidir:\n",
    "            encoder_states = []\n",
    "            layer_inputs = embed_inputs\n",
    "\n",
    "            for i in range(num_layers):\n",
    "                with tf.variable_scope(\"layer_%d\" % (i + 1)):\n",
    "                    fw_cell = build_rnn_cell(\n",
    "                        1, num_units, cell_type, forget_bias)\n",
    "                    bw_cell = build_rnn_cell(\n",
    "                        1, num_units, cell_type, forget_bias)\n",
    "\n",
    "                    outs = tf.nn.bidirectional_dynamic_rnn(\n",
    "                        fw_cell, bw_cell, layer_inputs,\n",
    "                        dtype=dtype,\n",
    "                        swap_memory=True)\n",
    "                    bi_outputs, (state_fw, state_bw) = outs\n",
    "\n",
    "                    if cell_type == \"LSTM\":\n",
    "                        state_c = state_fw.c + state_bw.c\n",
    "                        state_h = state_fw.h + state_bw.h\n",
    "                        encoder_states.append(LSTMStateTuple(state_c, state_h))\n",
    "                    else:\n",
    "                        encoder_states.append(state_fw + state_bw)\n",
    "\n",
    "                    if i < num_layers - 1:\n",
    "                        layer_inputs = tf.layers.dense(\n",
    "                            tf.concat(bi_outputs, -1), num_units)\n",
    "\n",
    "            if self_attention:\n",
    "                encoder_hs = tf.concat(bi_outputs, -1)\n",
    "                attn_scores = self_attention_scores(\n",
    "                    encoder_hs, num_attn_hidden)\n",
    "                output_h = tf.reduce_sum(attn_scores * encoder_hs, axis=1)\n",
    "\n",
    "            else:\n",
    "                fw_h = bi_outputs[0][:, -1, :]\n",
    "                bw_h = bi_outputs[1][:, 0, :]\n",
    "                output_h = tf.concat([fw_h, bw_h], -1)\n",
    "\n",
    "        # unidirectional\n",
    "        else:\n",
    "            rnn_cell = build_rnn_cell(\n",
    "                num_layers, num_units, cell_type, forget_bias)\n",
    "            encoder_outputs, encoder_states = tf.nn.dynamic_rnn(\n",
    "                rnn_cell, embed_inputs,\n",
    "                dtype=dtype,\n",
    "                swap_memory=True)\n",
    "\n",
    "            if self_attention:\n",
    "                attn_scores = self_attention_scores(\n",
    "                    encoder_outputs, num_attn_hidden)\n",
    "                output_h = tf.reduce_sum(attn_scores * encoder_hs, axis=-1)\n",
    "            else:\n",
    "                output_h = encoder_outputs[:, -1, :]\n",
    "\n",
    "        outputs = tf.nn.sigmoid(output_layer(output_h))\n",
    "\n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_embeddings(vocab_size, embed_size, dtype=tf.float32,\n",
    "                    initializer=None, initial_values=None,\n",
    "                    name='embeddings'):\n",
    "    \"\"\"\n",
    "    embeddings:\n",
    "        initialize trainable embeddings or load pretrained from files\n",
    "    \"\"\"\n",
    "    with tf.variable_scope(name):\n",
    "        if initial_values:\n",
    "            embeddings = tf.Variable(initial_value=initial_values,\n",
    "                                     name=\"embeddings\", dtype=dtype)\n",
    "        else:\n",
    "            if initializer is None:\n",
    "                initializer = tf.contrib.layers.xavier_initializer()\n",
    "            embeddings = tf.Variable(\n",
    "                initializer(shape=(vocab_size, embed_size)),\n",
    "                name=\"embeddings\", dtype=dtype)\n",
    "\n",
    "        # id_0 represents all-zero token, id_1 represents UNK token\n",
    "        zero_embed = tf.zeros(shape=[1, embed_size])\n",
    "        unk_embed = tf.get_variable(\"UNK\", [1, embed_size], dtype)\n",
    "        embeddings = tf.concat([zero_embed, unk_embed, embeddings], axis=0)\n",
    "\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embeddings = init_embeddings(1000, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_ids = tf.placeholder(tf.int32, [None, None])\n",
    "outputs = build_emotion_regressor(embeddings, source_ids, num_layers=2, num_units=256,\n",
    "                                  num_emotions=4, cell_type=\"LSTM\", bidir=True,\n",
    "                                  self_attention=True, num_attn_hidden=128, name=\"e3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute loss\n",
    "def compute_loss(source_ids, targets, embeddings, num_layers, num_units,\n",
    "                 num_emotions, cell_type, enc_bidir, self_attention=False,\n",
    "                 num_attn_hidden=128, l2_regularize=None, name=\"emo_reg\"):\n",
    "    \"\"\"\n",
    "    Creates a emotion regressor and returns squared loss.\n",
    "    \"\"\"\n",
    "    with tf.name_scope(name):\n",
    "        # build emotion regressor\n",
    "        outputs = build_emotion_regressor(\n",
    "            embeddings, source_ids, num_layers, num_units,\n",
    "            num_emotions, cell_type, bidir=enc_bidir,\n",
    "            self_attention=self_attention, num_attn_hidden=num_attn_hidden,\n",
    "            name=name)\n",
    "\n",
    "        # compute loss\n",
    "        with tf.name_scope('loss'):\n",
    "            reduced_loss = tf.losses.mean_squared_error(\n",
    "                labels=targets, predictions=outputs)\n",
    "\n",
    "            if l2_regularize is None:\n",
    "                return reduced_loss, outputs\n",
    "            else:\n",
    "                l2_loss = tf.add_n([tf.nn.l2_loss(v)\n",
    "                                    for v in tf.trainable_variables()\n",
    "                                    if not('bias' in v.name)])\n",
    "\n",
    "                total_loss = reduced_loss + l2_regularize * l2_loss\n",
    "                return total_loss, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = tf.placeholder(tf.float32, [None, 4])\n",
    "loss = tf.losses.mean_squared_error(labels=targets, predictions=outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving and load\n",
    "def load(saver, sess, logdir):\n",
    "    print(\"Trying to restore saved checkpoints from {} ...\".format(logdir),\n",
    "          end=\"\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(logdir)\n",
    "    if ckpt:\n",
    "        print(\"  Checkpoint found: {}\".format(ckpt.model_checkpoint_path))\n",
    "        global_step = int(ckpt.model_checkpoint_path\n",
    "                          .split('/')[-1]\n",
    "                          .split('-')[-1])\n",
    "        print(\"  Global step was: {}\".format(global_step))\n",
    "        print(\"  Restoring...\", end=\"\")\n",
    "        saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "        print(\" Done.\")\n",
    "        return global_step\n",
    "    else:\n",
    "        print(\" No checkpoint found.\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def save(saver, sess, logdir, step):\n",
    "    model_name = 'model.ckpt'\n",
    "    checkpoint_path = os.path.join(logdir, model_name)\n",
    "    print('Storing checkpoint to {} ...'.format(logdir), end=\"\")\n",
    "    sys.stdout.flush()\n",
    "\n",
    "    if not os.path.exists(logdir):\n",
    "        os.makedirs(logdir)\n",
    "\n",
    "    saver.save(sess, checkpoint_path, global_step=step)\n",
    "    print(' Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inference and Evaluation\n",
    "def eval_mean_pearson(source_ids, predictions, sess, data, labels):\n",
    "    \"\"\"\n",
    "    Compute Pearson's correlation coeff w.r.t. each emotion\n",
    "    and average all coeffs as the evaluation metric.\n",
    "    Ref: SemEval-2018 Task 1: Affect in Tweets (AIT-2018)\n",
    "    \"\"\"\n",
    "    pred = sess.run(predictions, feed_dict={source_ids: data})\n",
    "    pearsons = [pearsonr(pred[:, i], labels[:, i])[0] for i in range(4)]\n",
    "    mean_pearson = np.mean(pearsons)\n",
    "    return mean_pearson"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize\n",
    "def loadfilename(filename):\n",
    "    emotions = [\"anger\", \"fear\", \"joy\", \"sadness\"]\n",
    "    data = {}\n",
    "\n",
    "    for ith, emo in enumerate(emotions):\n",
    "        df = pd.read_csv(filename.format(emo), \"\\t\").values\n",
    "        for ID, tweet, _, score in df:\n",
    "            if ID not in data:\n",
    "                # emotion is a 4-dimensional real-valued vector\n",
    "                data[ID] = {\n",
    "                    \"text\": tweet,\n",
    "                    \"emotion\": np.zeros(4)\n",
    "                }\n",
    "\n",
    "            data[ID][\"emotion\"][ith] = float(score)\n",
    "\n",
    "    return data\n",
    "\n",
    "\n",
    "def tokenize(data):\n",
    "    \"\"\"\n",
    "    NLTK Tweet: Tokenizations for training data, build a dictionary\n",
    "        data: {ID: {text: tweet, emotion: vector}}\n",
    "    \"\"\"\n",
    "    twtknzr = TweetTokenizer(reduce_len=True, strip_handles=True)\n",
    "    words = {}\n",
    "    index = 0\n",
    "    tk_r = []\n",
    "    strings = []\n",
    "\n",
    "    # Build dictionary\n",
    "    for ID in data.keys():\n",
    "        tk_r.append(twtknzr.tokenize(data[ID][\"text\"]))\n",
    "        tk_ids = []\n",
    "\n",
    "        for word in tk_r[-1]:\n",
    "            # strip hashtag's #\n",
    "            if word[0] == \"#\":\n",
    "                word = word[1:]\n",
    "\n",
    "            if word not in words:\n",
    "                words[word] = index\n",
    "                index += 1\n",
    "\n",
    "            strings.append(word)\n",
    "            tk_ids.append(words[word])\n",
    "\n",
    "        # transform text into token_id sequence\n",
    "        data[ID][\"ids\"] = tk_ids\n",
    "\n",
    "    return words\n",
    "\n",
    "\n",
    "def tokenize_test(data, words):\n",
    "    \"\"\"\n",
    "    NLTK Tweet: Tokenizations for dev/valid/test data\n",
    "        data: {ID: {text: tweet, emotion: vector}}\n",
    "        words: predefined dictionary\n",
    "    \"\"\"\n",
    "    twtknzr = TweetTokenizer(reduce_len=True, strip_handles=True)\n",
    "    num_unk = 0\n",
    "    total_word = 0\n",
    "    tk_r = []\n",
    "\n",
    "    for ID in data.keys():\n",
    "        tk_r.append(twtknzr.tokenize(data[ID][\"text\"]))\n",
    "        tk_ids = []\n",
    "        for word in tk_r[-1]:\n",
    "            if word[0] == \"#\":\n",
    "                word = word[1:]\n",
    "\n",
    "            if word not in words:\n",
    "                num_unk += 1\n",
    "            else:\n",
    "                tk_ids.append(words[word])\n",
    "\n",
    "            total_word += 1\n",
    "\n",
    "        data[ID][\"ids\"] = tk_ids\n",
    "\n",
    "    return num_unk, total_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess\n",
    "def to_output_form(data):\n",
    "    \"\"\"\n",
    "    Transform data into a list of strings output format\n",
    "    \"\"\"\n",
    "    processed = []\n",
    "    for ID in data.keys():\n",
    "        text = \" \".join(np.array(data[ID][\"ids\"], dtype=str).tolist())\n",
    "        emo = \" \".join(data[ID][\"emotion\"].astype(str).tolist())\n",
    "        processed.append(text + \",\" + emo)\n",
    "\n",
    "    return processed\n",
    "\n",
    "\n",
    "def preprocess():\n",
    "    train_filename = \"./EI-reg-En-train/EI-reg-En-{}-train.txt\"\n",
    "    train_data = loadfilename(train_filename)\n",
    "    train_words = tokenize(train_data)\n",
    "    train_processed = to_output_form(train_data)\n",
    "\n",
    "    train_df = pd.DataFrame(data={\"0\": train_processed})\n",
    "    train_df.to_csv(\"emodata_train\", header=None, index=None)\n",
    "\n",
    "    word_df = pd.DataFrame(data={\"0\": train_words})\n",
    "    word_df.to_csv(\"emodata_word_ids\", header=None, sep=\"\\t\")\n",
    "    print(\"Training set processed. With {} words and {} sentences.\".format(\n",
    "          len(train_words), len(train_data)))\n",
    "\n",
    "    dev_filename = \"./2018-EI-reg-En-dev/2018-EI-reg-En-{}-dev.txt\"\n",
    "    dev_data = loadfilename(dev_filename)\n",
    "    num_unk, total_word = tokenize_test(dev_data, train_words)\n",
    "    dev_processed = to_output_form(dev_data)\n",
    "\n",
    "    dev_df = pd.DataFrame(data={\"0\": dev_processed})\n",
    "    dev_df.to_csv(\"emodata_dev\", header=None, index=None)\n",
    "    print(\"Dev set processed: {} UNKs, {} words and {} sentences.\".format(\n",
    "          num_unk, total_word, len(dev_data)))\n",
    "\n",
    "    return train_data, train_words, dev_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set processed. With 15448 words and 6908 sentences.\n",
      "Dev set processed: 1810 UNKs, 15694 words and 893 sentences.\n"
     ]
    }
   ],
   "source": [
    "train_data, train_words, dev_data = preprocess()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9470179502026635\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEYRJREFUeJzt3X+s3Xddx/Hny5bxU+l+XOZsi7dKI5mEH8vNmIGYuenoNkL3ByxbVAo2aYxDQTBQ8I9FDMmIxjkSXVJZpUtwsAxwDZtiM2bQxA3u+LWNgbuOjbbZ1ov7IUgAq2//OJ/qobS97Tn33rPez/ORnJzv9/39nPP9fLKzvs738/2e701VIUnqz09MugOSpMkwACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdWj3pDhzLGWecUdPT05PuhiSdVO65555vV9XUQu2e0QEwPT3N7OzspLshSSeVJI8cTzungCSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVPP6F8CSwuZ3n7bxPb98DWXTmzf0mLwCECSOmUASFKnDABJ6pQBIEmdWjAAkuxMciDJfUfY9q4kleSMtp4kH0oyl+SrSc4ZarslyYPtsWVxhyFJOlHHcwTwEWDT4cUk64GLgG8NlS8GNrbHNuD61vY04Grg1cC5wNVJTh2n45Kk8SwYAFX1OeCJI2y6Fng3UEO1zcCNNXAXsCbJWcDrgD1V9URVPQns4QihIklaPiP9DiDJZmB/VX0lyfCmtcDeofV9rXa0ulaISV6PL2k0JxwASZ4HvI/B9M+iS7KNwfQRL37xi5diF5IkRrsK6OeBDcBXkjwMrAO+mOSngf3A+qG261rtaPUfU1U7qmqmqmamphb8m8aSpBGdcABU1b1V9aKqmq6qaQbTOedU1WPAbuDN7Wqg84Cnq+pR4DPARUlObSd/L2o1SdKEHM9loDcB/wL8QpJ9SbYeo/ntwEPAHPBXwO8AVNUTwB8DX2iP97eaJGlCFjwHUFVXLrB9emi5gKuO0m4nsPME+ydJWiL+EliSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE75R+FXGG/KJul4eQQgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4tGABJdiY5kOS+odqfJPl6kq8m+VSSNUPb3ptkLsk3krxuqL6p1eaSbF/8oUiSTsTxHAF8BNh0WG0P8LKqejnwr8B7AZKcDVwB/GJ7zV8mWZVkFfAXwMXA2cCVra0kaUIWDICq+hzwxGG1f6iqg231LmBdW94MfKyqflBV3wTmgHPbY66qHqqqHwIfa20lSROyGOcAfgv4u7a8Ftg7tG1fqx2t/mOSbEsym2R2fn5+EbonSTqSsQIgyR8CB4GPLk53oKp2VNVMVc1MTU0t1ttKkg4z8l8ES/IW4PXAhVVVrbwfWD/UbF2rcYy6JGkCRjoCSLIJeDfwhqr63tCm3cAVSZ6dZAOwEfg88AVgY5INSU5hcKJ493hdlySNY8EjgCQ3AecDZyTZB1zN4KqfZwN7kgDcVVW/XVX3J7kZ+BqDqaGrquq/2/u8DfgMsArYWVX3L8F4JEnHacEAqKorj1C+4RjtPwB84Aj124HbT6h3kqQl4y+BJalTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnq1Mi3gtDRTW+/bdJdkKQFeQQgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1asEASLIzyYEk9w3VTkuyJ8mD7fnUVk+SDyWZS/LVJOcMvWZLa/9gki1LMxxJ0vE6niOAjwCbDqttB+6oqo3AHW0d4GJgY3tsA66HQWAAVwOvBs4Frj4UGpKkyVgwAKrqc8ATh5U3A7va8i7gsqH6jTVwF7AmyVnA64A9VfVEVT0J7OHHQ0WStIxGPQdwZlU92pYfA85sy2uBvUPt9rXa0eo/Jsm2JLNJZufn50fsniRpIWOfBK6qAmoR+nLo/XZU1UxVzUxNTS3W20qSDjNqADzepnZozwdafT+wfqjdulY7Wl2SNCGjBsBu4NCVPFuAW4fqb25XA50HPN2mij4DXJTk1Hby96JWkyRNyIJ/EjLJTcD5wBlJ9jG4muca4OYkW4FHgMtb89uBS4A54HvAWwGq6okkfwx8obV7f1UdfmJZkrSMFgyAqrryKJsuPELbAq46yvvsBHaeUO8kSUvGPwovjWh6+20T2e/D11w6kf1q5fFWEJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOjVWACT5/ST3J7kvyU1JnpNkQ5K7k8wl+XiSU1rbZ7f1ubZ9ejEGIEkazcgBkGQt8HvATFW9DFgFXAF8ELi2ql4CPAlsbS/ZCjzZ6te2dpKkCRl3Cmg18Nwkq4HnAY8CFwC3tO27gMva8ua2Ttt+YZKMuX9J0ohGDoCq2g/8KfAtBv/wPw3cAzxVVQdbs33A2ra8FtjbXnuwtT991P1LksYzzhTQqQy+1W8AfgZ4PrBp3A4l2ZZkNsns/Pz8uG8nSTqKcaaAfhX4ZlXNV9V/AZ8EXgOsaVNCAOuA/W15P7AeoG1/IfDvh79pVe2oqpmqmpmamhqje5KkYxknAL4FnJfkeW0u/0Lga8CdwBtbmy3ArW15d1unbf9sVdUY+5ckjWGccwB3MziZ+0Xg3vZeO4D3AO9MMsdgjv+G9pIbgNNb/Z3A9jH6LUka0+qFmxxdVV0NXH1Y+SHg3CO0/T7wpnH2J0laPP4SWJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASerUWAGQZE2SW5J8PckDSX4pyWlJ9iR5sD2f2tomyYeSzCX5apJzFmcIkqRRjHsEcB3w91X1UuAVwAPAduCOqtoI3NHWAS4GNrbHNuD6MfctSRrDyAGQ5IXALwM3AFTVD6vqKWAzsKs12wVc1pY3AzfWwF3AmiRnjdxzSdJYxjkC2ADMA3+d5EtJPpzk+cCZVfVoa/MYcGZbXgvsHXr9vlaTJE3AOAGwGjgHuL6qXgX8J/8/3QNAVRVQJ/KmSbYlmU0yOz8/P0b3JEnHMk4A7AP2VdXdbf0WBoHw+KGpnfZ8oG3fD6wfev26VvsRVbWjqmaqamZqamqM7kmSjmXkAKiqx4C9SX6hlS4EvgbsBra02hbg1ra8G3hzuxroPODpoakiSdIyWz3m638X+GiSU4CHgLcyCJWbk2wFHgEub21vBy4B5oDvtbaSpAkZKwCq6svAzBE2XXiEtgVcNc7+JEmLx18CS1KnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpU2MHQJJVSb6U5NNtfUOSu5PMJfl4klNa/dltfa5tnx5335Kk0S3GEcDbgQeG1j8IXFtVLwGeBLa2+lbgyVa/trWTJE3IWAGQZB1wKfDhth7gAuCW1mQXcFlb3tzWadsvbO0lSRMw7hHAnwPvBv6nrZ8OPFVVB9v6PmBtW14L7AVo259u7X9Ekm1JZpPMzs/Pj9k9SdLRrB71hUleDxyoqnuSnL9YHaqqHcAOgJmZmRrnvaa337YofZKklWjkAABeA7whySXAc4CfAq4D1iRZ3b7lrwP2t/b7gfXAviSrgRcC/z7G/iVJYxh5Cqiq3ltV66pqGrgC+GxV/TpwJ/DG1mwLcGtb3t3Wads/W1VjfcOXJI1uKX4H8B7gnUnmGMzx39DqNwCnt/o7ge1LsG9J0nEaZwro/1TVPwL/2JYfAs49QpvvA29ajP1JksbnL4ElqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTo38N4GTrAduBM4ECthRVdclOQ34ODANPAxcXlVPJglwHXAJ8D3gLVX1xfG6L/VnevttE9v3w9dcOrF9a/GNcwRwEHhXVZ0NnAdcleRsYDtwR1VtBO5o6wAXAxvbYxtw/Rj7liSNaeQAqKpHD32Dr6rvAA8Aa4HNwK7WbBdwWVveDNxYA3cBa5KcNXLPJUljWZRzAEmmgVcBdwNnVtWjbdNjDKaIYBAOe4detq/VJEkTMHYAJHkB8AngHVX1H8PbqqoYnB84kffblmQ2yez8/Py43ZMkHcVYAZDkWQz+8f9oVX2ylR8/NLXTng+0+n5g/dDL17Xaj6iqHVU1U1UzU1NT43RPknQMIwdAu6rnBuCBqvqzoU27gS1teQtw61D9zRk4D3h6aKpIkrTMRr4MFHgN8JvAvUm+3GrvA64Bbk6yFXgEuLxtu53BJaBzDC4DfesY+5YkjWnkAKiqfwZylM0XHqF9AVeNuj9J0uLyl8CS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjo1zt8EltSZ6e23TWS/D19z6UT2u9It+xFAkk1JvpFkLsn25d6/JGlgWQMgySrgL4CLgbOBK5OcvZx9kCQNLPcRwLnAXFU9VFU/BD4GbF7mPkiSWP5zAGuBvUPr+4BXL3MfJJ1kJnXuAVb2+Ydn3EngJNuAbW31u0m+scBLzgC+vbS9esbqdeyOuy8THXc+OKk9jzXunz2eRssdAPuB9UPr61rt/1TVDmDH8b5hktmqmlmc7p1ceh274+6L4146y30O4AvAxiQbkpwCXAHsXuY+SJJY5iOAqjqY5G3AZ4BVwM6qun85+yBJGlj2cwBVdTtw+yK+5XFPF61AvY7dcffFcS+RVNVS70OS9AzkvYAkqVMndQD0cluJJDuTHEhy31DttCR7kjzYnk+dZB+XQpL1Se5M8rUk9yd5e6uv6LEneU6Szyf5Shv3H7X6hiR3t8/7x9uFFCtOklVJvpTk0229l3E/nOTeJF9OMttqS/pZP2kDoLPbSnwE2HRYbTtwR1VtBO5o6yvNQeBdVXU2cB5wVftvvNLH/gPggqp6BfBKYFOS84APAtdW1UuAJ4GtE+zjUno78MDQei/jBviVqnrl0OWfS/pZP2kDgI5uK1FVnwOeOKy8GdjVlncBly1rp5ZBVT1aVV9sy99h8I/CWlb42Gvgu231We1RwAXALa2+4sYNkGQdcCnw4bYeOhj3MSzpZ/1kDoAj3VZi7YT6MglnVtWjbfkx4MxJdmapJZkGXgXcTQdjb9MgXwYOAHuAfwOeqqqDrclK/bz/OfBu4H/a+un0MW4YhPw/JLmn3REBlviz/oy7FYROXFVVkhV7OVeSFwCfAN5RVf8x+FI4sFLHXlX/DbwyyRrgU8BLJ9ylJZfk9cCBqronyfmT7s8EvLaq9id5EbAnydeHNy7FZ/1kPgJY8LYSK9zjSc4CaM8HJtyfJZHkWQz+8f9oVX2ylbsYO0BVPQXcCfwSsCbJoS9tK/Hz/hrgDUkeZjClewFwHSt/3ABU1f72fIBB6J/LEn/WT+YA6P22EruBLW15C3DrBPuyJNr87w3AA1X1Z0ObVvTYk0y1b/4keS7wawzOf9wJvLE1W3Hjrqr3VtW6qppm8P/zZ6vq11nh4wZI8vwkP3loGbgIuI8l/qyf1D8ES3IJgznDQ7eV+MCEu7QkktwEnM/g7oCPA1cDfwvcDLwYeAS4vKoOP1F8UkvyWuCfgHv5/znh9zE4D7Bix57k5QxO+K1i8CXt5qp6f5KfY/DN+DTgS8BvVNUPJtfTpdOmgP6gql7fw7jbGD/VVlcDf1NVH0hyOkv4WT+pA0CSNLqTeQpIkjQGA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE79L9Hi+A0Jo3fRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# parsed sentence length distribution\n",
    "ls = np.array([len(train_data[ID][\"ids\"]) for ID in train_data.keys()])\n",
    "\n",
    "plt.hist(ls)\n",
    "print((ls < 30).sum() / float(len(ls)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# glove = pd.read_csv(\"glove.twitter.27B.100d.txt\", \" \", error_bad_lines=True, header=None, index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training: BiLSTM + SelfAttn, RanInit, Direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadfile(filename, max_length):\n",
    "    df = pd.read_csv(filename, header=None)\n",
    "    data = []\n",
    "    labels = []\n",
    "    for row in df[0].values:\n",
    "        row = row.split(\",\")\n",
    "        if len(row) == 2:\n",
    "            ids, emos = row\n",
    "            ids = np.array(ids.split(\" \"), dtype=np.int32)\n",
    "            emos = np.array(emos.split(\" \"), dtype=np.float)\n",
    "        else:\n",
    "            ids = np.array(row[0].split(\" \"), dtype=np.float)\n",
    "            emos = None\n",
    "\n",
    "        if len(ids) < max_length:\n",
    "            # represents constant zero padding\n",
    "            pads = -np.ones(max_length - len(ids), dtype=np.int32)\n",
    "            ids = np.concatenate((ids, pads))\n",
    "        else:\n",
    "            ids = ids[:max_length]\n",
    "\n",
    "        data.append(ids)\n",
    "        labels.append(emos)\n",
    "\n",
    "    return np.array(data), np.array(labels)\n",
    "\n",
    "\n",
    "def get_config(config):\n",
    "    num_layers = config[\"model\"][\"num_layers\"]\n",
    "    num_units = config[\"model\"][\"num_units\"]\n",
    "    num_emotions = config[\"model\"][\"num_emotions\"]\n",
    "    cell_type = config[\"model\"][\"cell_type\"]\n",
    "    enc_bidir = config[\"model\"][\"bidirectional\"]\n",
    "    self_attention = config[\"model\"][\"self_attention\"]\n",
    "    num_attn_hidden = config[\"model\"][\"num_attn_hidden\"]\n",
    "\n",
    "    # infer_batch_size = config[\"inference\"][\"infer_batch_size\"]\n",
    "    # infer_type = config[\"inference\"][\"type\"]\n",
    "    # beam_size = config[\"inference\"][\"beam_size\"]\n",
    "    # max_iter = config[\"inference\"][\"max_length\"]\n",
    "\n",
    "    train_config = config[\"training\"]\n",
    "    logdir = train_config[\"logdir\"]\n",
    "    restore_from = train_config[\"restore_from\"]\n",
    "    l2_regularize = train_config[\"l2_regularize\"]\n",
    "\n",
    "    learning_rate = train_config[\"learning_rate\"]\n",
    "    gpu_fraction = train_config[\"gpu_fraction\"]\n",
    "    max_checkpoints = train_config[\"max_checkpoints\"]\n",
    "    train_steps = train_config[\"train_steps\"]\n",
    "    batch_size = train_config[\"batch_size\"]\n",
    "    print_every = train_config[\"print_every\"]\n",
    "    checkpoint_every = train_config[\"checkpoint_every\"]\n",
    "\n",
    "    loss_fig = train_config[\"loss_fig\"]\n",
    "    pearson_fig = train_config[\"pearson_fig\"]\n",
    "\n",
    "    return (num_layers, num_units, num_emotions, cell_type, enc_bidir,\n",
    "            self_attention, num_attn_hidden, logdir, restore_from,\n",
    "            l2_regularize, learning_rate, gpu_fraction, max_checkpoints,\n",
    "            train_steps, batch_size, print_every, checkpoint_every,\n",
    "            loss_fig, pearson_fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "\n",
    "config_details = {\n",
    "    \"Name\": \"BiLSTM_SelfAttn_EmoRegressor\",\n",
    "    \"embeddings\": {\n",
    "        \"embed_size\": 128,\n",
    "    },\n",
    "    \"model\": {\n",
    "        \"num_layers\": 2,\n",
    "        \"num_units\": 256,\n",
    "        \"num_emotions\": 4,\n",
    "        \"cell_type\": \"LSTM\",\n",
    "        \"bidirectional\": True,\n",
    "        \"self_attention\": True,\n",
    "        \"num_attn_hidden\": 128,\n",
    "    },    \n",
    "    \"inference\": {\n",
    "        \"infer_batch_size\": 15,\n",
    "        \"infer_source_file\": \"./emodata_dev\",\n",
    "        \"infer_source_max_length\": 25,\n",
    "        \"output_path\": \"./emoprediction.txt\",\n",
    "    },\n",
    "    \"training\": {\n",
    "        \"l2_regularize\": None,\n",
    "        \"logdir\": \"./log_emoReg/\",\n",
    "        \"restore_from\": \"./log_emoReg/\",\n",
    "        \"learning_rate\": 1e-3,\n",
    "        \"batch_size\": 64,\n",
    "        \"gpu_fraction\": 0.05,\n",
    "        \"max_checkpoints\": 10000,\n",
    "        \"train_steps\": 4000,\n",
    "        \"print_every\": 20,\n",
    "        \"checkpoint_every\": 500,\n",
    "        \"max_length\": 25,\n",
    "        \"loss_fig\": \"./training_loss_over_time\",\n",
    "        \"pearson_fig\": \"./pearson_over_time\",\n",
    "    }\n",
    "}\n",
    "\n",
    "with open('config_emoBiLSTM_selfattn.yaml', \"w\") as f:\n",
    "    yaml.dump({\"configuration\": config_details}, f, default_flow_style=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('config_emoBiLSTM_selfattn.yaml') as f:\n",
    "    # use safe_load instead load\n",
    "    config = yaml.safe_load(f)[\"configuration\"]\n",
    "\n",
    "name = config[\"Name\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data ...\n",
      "\tDone.\n",
      "Initializing embeddings ...\n",
      "\tDone.\n",
      "Building model architecture ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "# Load Data\n",
    "print(\"Loading data ...\")\n",
    "max_length = config[\"training\"][\"max_length\"]\n",
    "train_data, train_labels = loadfile(\"./emodata_train\", max_length)\n",
    "dev_data, dev_labels = loadfile(\"./emodata_dev\", max_length)\n",
    "word_ids = pd.read_csv(\"emodata_word_ids\", header=None, sep=\"\\t\")\n",
    "\n",
    "# id_0 represents all-zero token\n",
    "embed_shift = 1\n",
    "train_data += embed_shift\n",
    "dev_data += embed_shift\n",
    "n_data = len(train_data)\n",
    "print(\"\\tDone.\")\n",
    "\n",
    "# Construct or load embeddings\n",
    "print(\"Initializing embeddings ...\")\n",
    "vocab_size = len(word_ids)\n",
    "embed_size = config[\"embeddings\"][\"embed_size\"]\n",
    "embeddings = init_embeddings(vocab_size, embed_size, name=name)\n",
    "print(\"\\tDone.\")\n",
    "\n",
    "(num_layers, num_units, num_emotions, cell_type, enc_bidir, self_attention,\n",
    " num_attn_hidden, logdir, restore_from, l2_regularize, learning_rate,\n",
    " gpu_fraction, max_checkpoints, train_steps, batch_size, print_every,\n",
    " checkpoint_every, loss_fig, pearson_fig) = get_config(config)\n",
    "\n",
    "# Build the model and compute losses\n",
    "source_ids = tf.placeholder(tf.int32, [None, None], name=\"source\")\n",
    "targets = tf.placeholder(tf.float32, [None, num_emotions], name=\"target\")\n",
    "\n",
    "print(\"Building model architecture ...\")\n",
    "loss, predictions = compute_loss(\n",
    "    source_ids, targets, embeddings, num_layers, num_units,\n",
    "    num_emotions, cell_type, enc_bidir, self_attention,\n",
    "    num_attn_hidden, l2_regularize, name)\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to restore saved checkpoints from ./log_emoReg/ ... No checkpoint found.\n"
     ]
    }
   ],
   "source": [
    "# Even if we restored the model, we will treat it as new training\n",
    "# if the trained model is written into an arbitrary location.\n",
    "is_overwritten_training = logdir != restore_from\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,\n",
    "                                   epsilon=1e-4)\n",
    "trainable = tf.trainable_variables()\n",
    "optim = optimizer.minimize(loss, var_list=trainable)\n",
    "\n",
    "# Set up session\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_fraction)\n",
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,\n",
    "                                        gpu_options=gpu_options))\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "\n",
    "# Saver for storing checkpoints of the model.\n",
    "saver = tf.train.Saver(var_list=tf.trainable_variables(),\n",
    "                       max_to_keep=max_checkpoints)\n",
    "\n",
    "try:\n",
    "    saved_global_step = load(saver, sess, restore_from)\n",
    "    if is_overwritten_training or saved_global_step is None:\n",
    "        # The first training step will be saved_global_step + 1,\n",
    "        # therefore we put -1 here for new or overwritten trainings.\n",
    "        saved_global_step = -1\n",
    "\n",
    "except Exception:\n",
    "    print(\"Something went wrong while restoring checkpoint. \"\n",
    "          \"Training is terminated to avoid the overwriting.\")\n",
    "    raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training ...\n",
      "step 0 - loss = 0.19550, dev_loss = 0.15968, r = 0.0060, dev_r = 0.0015, (0.416 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 20 - loss = 0.08551, dev_loss = 0.07890, r = -0.0688, dev_r = -0.0015, (0.097 sec/step)\n",
      "step 40 - loss = 0.06926, dev_loss = 0.09955, r = -0.0044, dev_r = -0.0008, (0.096 sec/step)\n",
      "step 60 - loss = 0.05220, dev_loss = 0.08167, r = 0.0029, dev_r = 0.0006, (0.077 sec/step)\n",
      "step 80 - loss = 0.05985, dev_loss = 0.08139, r = 0.0801, dev_r = 0.0011, (0.106 sec/step)\n",
      "step 100 - loss = 0.06005, dev_loss = 0.08156, r = 0.0020, dev_r = 0.0026, (0.093 sec/step)\n",
      "step 120 - loss = 0.05175, dev_loss = 0.08232, r = 0.0569, dev_r = 0.0032, (0.082 sec/step)\n",
      "step 140 - loss = 0.05733, dev_loss = 0.08082, r = -0.0126, dev_r = 0.0053, (0.091 sec/step)\n",
      "step 160 - loss = 0.05650, dev_loss = 0.08235, r = 0.0561, dev_r = 0.0066, (0.077 sec/step)\n",
      "step 180 - loss = 0.05316, dev_loss = 0.08115, r = 0.0435, dev_r = 0.0081, (0.101 sec/step)\n",
      "step 200 - loss = 0.05773, dev_loss = 0.08143, r = -0.0173, dev_r = 0.0098, (0.096 sec/step)\n",
      "step 220 - loss = 0.06017, dev_loss = 0.08221, r = 0.0170, dev_r = 0.0119, (0.077 sec/step)\n",
      "step 240 - loss = 0.05829, dev_loss = 0.08166, r = 0.0104, dev_r = 0.0136, (0.089 sec/step)\n",
      "step 260 - loss = 0.05680, dev_loss = 0.08170, r = 0.0029, dev_r = 0.0149, (0.101 sec/step)\n",
      "step 280 - loss = 0.05650, dev_loss = 0.08197, r = 0.0697, dev_r = 0.0172, (0.092 sec/step)\n",
      "step 300 - loss = 0.05330, dev_loss = 0.08142, r = 0.0222, dev_r = 0.0184, (0.085 sec/step)\n",
      "step 320 - loss = 0.06325, dev_loss = 0.08238, r = 0.0610, dev_r = 0.0198, (0.095 sec/step)\n",
      "step 340 - loss = 0.06029, dev_loss = 0.08103, r = 0.0224, dev_r = 0.0221, (0.083 sec/step)\n",
      "step 360 - loss = 0.04928, dev_loss = 0.08168, r = 0.0900, dev_r = 0.0243, (0.098 sec/step)\n",
      "step 380 - loss = 0.05398, dev_loss = 0.08164, r = -0.0224, dev_r = 0.0259, (0.113 sec/step)\n",
      "step 400 - loss = 0.05807, dev_loss = 0.08300, r = 0.0104, dev_r = 0.0278, (0.099 sec/step)\n",
      "step 420 - loss = 0.05389, dev_loss = 0.08167, r = 0.0680, dev_r = 0.0303, (0.098 sec/step)\n",
      "step 440 - loss = 0.06412, dev_loss = 0.08143, r = 0.0874, dev_r = 0.0316, (0.077 sec/step)\n",
      "step 460 - loss = 0.05511, dev_loss = 0.08228, r = 0.0945, dev_r = 0.0335, (0.096 sec/step)\n",
      "step 480 - loss = 0.05838, dev_loss = 0.08092, r = 0.0976, dev_r = 0.0340, (0.080 sec/step)\n",
      "step 500 - loss = 0.05587, dev_loss = 0.08119, r = 0.0635, dev_r = 0.0350, (0.106 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 520 - loss = 0.04911, dev_loss = 0.08153, r = 0.0692, dev_r = 0.0357, (0.089 sec/step)\n",
      "step 540 - loss = 0.05644, dev_loss = 0.08096, r = 0.0451, dev_r = 0.0376, (0.126 sec/step)\n",
      "step 560 - loss = 0.06567, dev_loss = 0.08238, r = 0.1266, dev_r = 0.0391, (0.086 sec/step)\n",
      "step 580 - loss = 0.05697, dev_loss = 0.08315, r = 0.0799, dev_r = 0.0404, (0.083 sec/step)\n",
      "step 600 - loss = 0.06813, dev_loss = 0.08135, r = 0.1831, dev_r = 0.0415, (0.114 sec/step)\n",
      "step 620 - loss = 0.06054, dev_loss = 0.08150, r = 0.0884, dev_r = 0.0423, (0.085 sec/step)\n",
      "step 640 - loss = 0.05528, dev_loss = 0.08204, r = 0.0746, dev_r = 0.0419, (0.096 sec/step)\n",
      "step 660 - loss = 0.05613, dev_loss = 0.08163, r = 0.0524, dev_r = 0.0422, (0.093 sec/step)\n",
      "step 680 - loss = 0.05839, dev_loss = 0.08199, r = 0.1100, dev_r = 0.0426, (0.077 sec/step)\n",
      "step 700 - loss = 0.05178, dev_loss = 0.08215, r = 0.0248, dev_r = 0.0423, (0.077 sec/step)\n",
      "step 720 - loss = 0.05702, dev_loss = 0.08302, r = 0.0963, dev_r = 0.0421, (0.086 sec/step)\n",
      "step 740 - loss = 0.05771, dev_loss = 0.08117, r = 0.1124, dev_r = 0.0416, (0.085 sec/step)\n",
      "step 760 - loss = 0.05885, dev_loss = 0.08184, r = 0.1485, dev_r = 0.0431, (0.090 sec/step)\n",
      "step 780 - loss = 0.05764, dev_loss = 0.08025, r = 0.0901, dev_r = 0.0439, (0.099 sec/step)\n",
      "step 800 - loss = 0.06157, dev_loss = 0.08173, r = 0.0875, dev_r = 0.0450, (0.101 sec/step)\n",
      "step 820 - loss = 0.06390, dev_loss = 0.08125, r = 0.1482, dev_r = 0.0452, (0.103 sec/step)\n",
      "step 840 - loss = 0.05503, dev_loss = 0.08229, r = 0.0633, dev_r = 0.0450, (0.102 sec/step)\n",
      "step 860 - loss = 0.06043, dev_loss = 0.08091, r = 0.1360, dev_r = 0.0456, (0.103 sec/step)\n",
      "step 880 - loss = 0.05537, dev_loss = 0.08131, r = 0.0753, dev_r = 0.0448, (0.077 sec/step)\n",
      "step 900 - loss = 0.05612, dev_loss = 0.08062, r = 0.0486, dev_r = 0.0442, (0.078 sec/step)\n",
      "step 920 - loss = 0.05814, dev_loss = 0.08282, r = 0.1052, dev_r = 0.0445, (0.077 sec/step)\n",
      "step 940 - loss = 0.04856, dev_loss = 0.08379, r = 0.0999, dev_r = 0.0440, (0.104 sec/step)\n",
      "step 960 - loss = 0.05466, dev_loss = 0.08124, r = 0.0606, dev_r = 0.0450, (0.116 sec/step)\n",
      "step 980 - loss = 0.06063, dev_loss = 0.08225, r = 0.1151, dev_r = 0.0445, (0.084 sec/step)\n",
      "step 1000 - loss = 0.05211, dev_loss = 0.08270, r = 0.1177, dev_r = 0.0440, (0.079 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 1020 - loss = 0.04940, dev_loss = 0.08218, r = 0.0965, dev_r = 0.0434, (0.092 sec/step)\n",
      "step 1040 - loss = 0.06664, dev_loss = 0.08105, r = 0.2180, dev_r = 0.0431, (0.106 sec/step)\n",
      "step 1060 - loss = 0.05117, dev_loss = 0.08222, r = 0.0720, dev_r = 0.0431, (0.084 sec/step)\n",
      "step 1080 - loss = 0.05384, dev_loss = 0.08155, r = 0.1255, dev_r = 0.0420, (0.118 sec/step)\n",
      "step 1100 - loss = 0.05771, dev_loss = 0.08149, r = 0.1039, dev_r = 0.0426, (0.086 sec/step)\n",
      "step 1120 - loss = 0.05757, dev_loss = 0.08150, r = 0.0996, dev_r = 0.0420, (0.078 sec/step)\n",
      "step 1140 - loss = 0.05757, dev_loss = 0.08263, r = 0.0513, dev_r = 0.0424, (0.096 sec/step)\n",
      "step 1160 - loss = 0.04975, dev_loss = 0.08174, r = 0.0620, dev_r = 0.0414, (0.080 sec/step)\n",
      "step 1180 - loss = 0.06207, dev_loss = 0.08192, r = 0.0948, dev_r = 0.0421, (0.092 sec/step)\n",
      "step 1200 - loss = 0.05069, dev_loss = 0.08322, r = 0.0700, dev_r = 0.0430, (0.094 sec/step)\n",
      "step 1220 - loss = 0.05994, dev_loss = 0.08198, r = 0.0932, dev_r = 0.0433, (0.078 sec/step)\n",
      "step 1240 - loss = 0.04830, dev_loss = 0.08231, r = 0.1172, dev_r = 0.0439, (0.120 sec/step)\n",
      "step 1260 - loss = 0.06689, dev_loss = 0.08187, r = 0.1018, dev_r = 0.0444, (0.116 sec/step)\n",
      "step 1280 - loss = 0.06594, dev_loss = 0.08052, r = 0.0565, dev_r = 0.0446, (0.094 sec/step)\n",
      "step 1300 - loss = 0.05321, dev_loss = 0.07977, r = 0.0938, dev_r = 0.0450, (0.089 sec/step)\n",
      "step 1320 - loss = 0.05500, dev_loss = 0.08030, r = 0.1444, dev_r = 0.0460, (0.081 sec/step)\n",
      "step 1340 - loss = 0.05789, dev_loss = 0.08216, r = 0.1549, dev_r = 0.0462, (0.091 sec/step)\n",
      "step 1360 - loss = 0.06239, dev_loss = 0.08150, r = 0.1520, dev_r = 0.0469, (0.078 sec/step)\n",
      "step 1380 - loss = 0.06718, dev_loss = 0.08136, r = 0.0846, dev_r = 0.0471, (0.081 sec/step)\n",
      "step 1400 - loss = 0.05274, dev_loss = 0.08166, r = 0.0929, dev_r = 0.0479, (0.080 sec/step)\n",
      "step 1420 - loss = 0.04812, dev_loss = 0.08359, r = 0.0452, dev_r = 0.0480, (0.085 sec/step)\n",
      "step 1440 - loss = 0.05812, dev_loss = 0.08228, r = 0.1059, dev_r = 0.0474, (0.091 sec/step)\n",
      "step 1460 - loss = 0.05948, dev_loss = 0.08288, r = 0.0743, dev_r = 0.0478, (0.110 sec/step)\n",
      "step 1480 - loss = 0.06285, dev_loss = 0.08115, r = 0.1515, dev_r = 0.0490, (0.091 sec/step)\n",
      "step 1500 - loss = 0.06063, dev_loss = 0.08205, r = 0.1840, dev_r = 0.0500, (0.083 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 1520 - loss = 0.05493, dev_loss = 0.08284, r = 0.1280, dev_r = 0.0508, (0.097 sec/step)\n",
      "step 1540 - loss = 0.05237, dev_loss = 0.08094, r = 0.0905, dev_r = 0.0515, (0.106 sec/step)\n",
      "step 1560 - loss = 0.05253, dev_loss = 0.08119, r = 0.1335, dev_r = 0.0516, (0.103 sec/step)\n",
      "step 1580 - loss = 0.05750, dev_loss = 0.08170, r = 0.1247, dev_r = 0.0521, (0.100 sec/step)\n",
      "step 1600 - loss = 0.06260, dev_loss = 0.08040, r = 0.0887, dev_r = 0.0536, (0.081 sec/step)\n",
      "step 1620 - loss = 0.06213, dev_loss = 0.08029, r = 0.1166, dev_r = 0.0545, (0.095 sec/step)\n",
      "step 1640 - loss = 0.06330, dev_loss = 0.08084, r = 0.1378, dev_r = 0.0549, (0.104 sec/step)\n",
      "step 1660 - loss = 0.05459, dev_loss = 0.08224, r = 0.0984, dev_r = 0.0567, (0.087 sec/step)\n",
      "step 1680 - loss = 0.05398, dev_loss = 0.08266, r = 0.0913, dev_r = 0.0579, (0.087 sec/step)\n",
      "step 1700 - loss = 0.04784, dev_loss = 0.08170, r = 0.1087, dev_r = 0.0580, (0.083 sec/step)\n",
      "step 1720 - loss = 0.05506, dev_loss = 0.08146, r = 0.1096, dev_r = 0.0601, (0.101 sec/step)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 1740 - loss = 0.05249, dev_loss = 0.08101, r = 0.0746, dev_r = 0.0605, (0.088 sec/step)\n",
      "step 1760 - loss = 0.05741, dev_loss = 0.08175, r = 0.0946, dev_r = 0.0621, (0.079 sec/step)\n",
      "step 1780 - loss = 0.05658, dev_loss = 0.08185, r = 0.0714, dev_r = 0.0640, (0.090 sec/step)\n",
      "step 1800 - loss = 0.06029, dev_loss = 0.08246, r = 0.1030, dev_r = 0.0671, (0.109 sec/step)\n",
      "step 1820 - loss = 0.06069, dev_loss = 0.08053, r = 0.0649, dev_r = 0.0680, (0.086 sec/step)\n",
      "step 1840 - loss = 0.05027, dev_loss = 0.08181, r = 0.1441, dev_r = 0.0723, (0.078 sec/step)\n",
      "step 1860 - loss = 0.06029, dev_loss = 0.08111, r = 0.1422, dev_r = 0.0772, (0.087 sec/step)\n",
      "step 1880 - loss = 0.06821, dev_loss = 0.08325, r = 0.2288, dev_r = 0.0828, (0.085 sec/step)\n",
      "step 1900 - loss = 0.05466, dev_loss = 0.08067, r = 0.1574, dev_r = 0.0878, (0.095 sec/step)\n",
      "step 1920 - loss = 0.05645, dev_loss = 0.08055, r = 0.2242, dev_r = 0.1068, (0.083 sec/step)\n",
      "step 1940 - loss = 0.05580, dev_loss = 0.08036, r = 0.1329, dev_r = 0.1333, (0.128 sec/step)\n",
      "step 1960 - loss = 0.05897, dev_loss = 0.08077, r = 0.2648, dev_r = 0.1607, (0.131 sec/step)\n",
      "step 1980 - loss = 0.05959, dev_loss = 0.08510, r = 0.1777, dev_r = 0.2219, (0.086 sec/step)\n",
      "step 2000 - loss = 0.05511, dev_loss = 0.07907, r = 0.2296, dev_r = 0.2712, (0.098 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 2020 - loss = 0.04112, dev_loss = 0.07795, r = 0.2673, dev_r = 0.2706, (0.085 sec/step)\n",
      "step 2040 - loss = 0.04021, dev_loss = 0.07395, r = 0.3264, dev_r = 0.2824, (0.080 sec/step)\n",
      "step 2060 - loss = 0.03995, dev_loss = 0.07583, r = 0.3912, dev_r = 0.2966, (0.114 sec/step)\n",
      "step 2080 - loss = 0.04453, dev_loss = 0.07681, r = 0.5377, dev_r = 0.2811, (0.090 sec/step)\n",
      "step 2100 - loss = 0.03970, dev_loss = 0.07383, r = 0.5619, dev_r = 0.3272, (0.098 sec/step)\n",
      "step 2120 - loss = 0.03017, dev_loss = 0.07346, r = 0.6037, dev_r = 0.3086, (0.087 sec/step)\n",
      "step 2140 - loss = 0.02870, dev_loss = 0.07146, r = 0.6260, dev_r = 0.2866, (0.079 sec/step)\n",
      "step 2160 - loss = 0.03706, dev_loss = 0.07572, r = 0.6065, dev_r = 0.3366, (0.103 sec/step)\n",
      "step 2180 - loss = 0.03401, dev_loss = 0.07220, r = 0.5955, dev_r = 0.2976, (0.091 sec/step)\n",
      "step 2200 - loss = 0.03124, dev_loss = 0.07289, r = 0.6403, dev_r = 0.3359, (0.103 sec/step)\n",
      "step 2220 - loss = 0.03473, dev_loss = 0.07370, r = 0.6838, dev_r = 0.3246, (0.118 sec/step)\n",
      "step 2240 - loss = 0.03434, dev_loss = 0.07373, r = 0.6223, dev_r = 0.3155, (0.124 sec/step)\n",
      "step 2260 - loss = 0.02734, dev_loss = 0.07329, r = 0.6606, dev_r = 0.3332, (0.096 sec/step)\n",
      "step 2280 - loss = 0.02434, dev_loss = 0.07166, r = 0.7081, dev_r = 0.3315, (0.086 sec/step)\n",
      "step 2300 - loss = 0.03658, dev_loss = 0.07352, r = 0.6922, dev_r = 0.3280, (0.077 sec/step)\n",
      "step 2320 - loss = 0.03270, dev_loss = 0.07319, r = 0.7107, dev_r = 0.3417, (0.086 sec/step)\n",
      "step 2340 - loss = 0.02757, dev_loss = 0.07547, r = 0.6747, dev_r = 0.3573, (0.110 sec/step)\n",
      "step 2360 - loss = 0.02771, dev_loss = 0.07044, r = 0.7222, dev_r = 0.3694, (0.083 sec/step)\n",
      "step 2380 - loss = 0.02839, dev_loss = 0.07288, r = 0.7272, dev_r = 0.3617, (0.082 sec/step)\n",
      "step 2400 - loss = 0.03279, dev_loss = 0.07919, r = 0.6830, dev_r = 0.2984, (0.103 sec/step)\n",
      "step 2420 - loss = 0.02601, dev_loss = 0.07402, r = 0.6649, dev_r = 0.3418, (0.101 sec/step)\n",
      "step 2440 - loss = 0.02323, dev_loss = 0.07652, r = 0.8398, dev_r = 0.3527, (0.097 sec/step)\n",
      "step 2460 - loss = 0.03323, dev_loss = 0.07844, r = 0.7150, dev_r = 0.3307, (0.089 sec/step)\n",
      "step 2480 - loss = 0.02427, dev_loss = 0.07802, r = 0.7680, dev_r = 0.3236, (0.087 sec/step)\n",
      "step 2500 - loss = 0.02397, dev_loss = 0.07662, r = 0.7761, dev_r = 0.3175, (0.078 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 2520 - loss = 0.02200, dev_loss = 0.07938, r = 0.8033, dev_r = 0.3305, (0.096 sec/step)\n",
      "step 2540 - loss = 0.02604, dev_loss = 0.07788, r = 0.7284, dev_r = 0.3429, (0.078 sec/step)\n",
      "step 2560 - loss = 0.02708, dev_loss = 0.07864, r = 0.8293, dev_r = 0.3233, (0.101 sec/step)\n",
      "step 2580 - loss = 0.02127, dev_loss = 0.07960, r = 0.7682, dev_r = 0.3256, (0.099 sec/step)\n",
      "step 2600 - loss = 0.02195, dev_loss = 0.07746, r = 0.8006, dev_r = 0.3350, (0.085 sec/step)\n",
      "step 2620 - loss = 0.01976, dev_loss = 0.07557, r = 0.8618, dev_r = 0.3674, (0.094 sec/step)\n",
      "step 2640 - loss = 0.01694, dev_loss = 0.07459, r = 0.8465, dev_r = 0.3711, (0.090 sec/step)\n",
      "step 2660 - loss = 0.01395, dev_loss = 0.07495, r = 0.9074, dev_r = 0.3760, (0.118 sec/step)\n",
      "step 2680 - loss = 0.02114, dev_loss = 0.07626, r = 0.8257, dev_r = 0.3749, (0.082 sec/step)\n",
      "step 2700 - loss = 0.01868, dev_loss = 0.07730, r = 0.8751, dev_r = 0.3891, (0.098 sec/step)\n",
      "step 2720 - loss = 0.02010, dev_loss = 0.07530, r = 0.8277, dev_r = 0.4006, (0.085 sec/step)\n",
      "step 2740 - loss = 0.01831, dev_loss = 0.07821, r = 0.8256, dev_r = 0.3675, (0.082 sec/step)\n",
      "step 2760 - loss = 0.02113, dev_loss = 0.07605, r = 0.8532, dev_r = 0.3972, (0.094 sec/step)\n",
      "step 2780 - loss = 0.01357, dev_loss = 0.07366, r = 0.9019, dev_r = 0.4156, (0.085 sec/step)\n",
      "step 2800 - loss = 0.01265, dev_loss = 0.07575, r = 0.9115, dev_r = 0.4005, (0.091 sec/step)\n",
      "step 2820 - loss = 0.01432, dev_loss = 0.07377, r = 0.9129, dev_r = 0.4175, (0.094 sec/step)\n",
      "step 2840 - loss = 0.01543, dev_loss = 0.07670, r = 0.8991, dev_r = 0.3862, (0.081 sec/step)\n",
      "step 2860 - loss = 0.01850, dev_loss = 0.07678, r = 0.8864, dev_r = 0.3942, (0.090 sec/step)\n",
      "step 2880 - loss = 0.01626, dev_loss = 0.07521, r = 0.9415, dev_r = 0.4089, (0.083 sec/step)\n",
      "step 2900 - loss = 0.01177, dev_loss = 0.07643, r = 0.9162, dev_r = 0.4185, (0.080 sec/step)\n",
      "step 2920 - loss = 0.01453, dev_loss = 0.07533, r = 0.8761, dev_r = 0.4220, (0.093 sec/step)\n",
      "step 2940 - loss = 0.01672, dev_loss = 0.07547, r = 0.9003, dev_r = 0.4189, (0.084 sec/step)\n",
      "step 2960 - loss = 0.01128, dev_loss = 0.07560, r = 0.8953, dev_r = 0.4243, (0.079 sec/step)\n",
      "step 2980 - loss = 0.00972, dev_loss = 0.07562, r = 0.9546, dev_r = 0.4151, (0.078 sec/step)\n",
      "step 3000 - loss = 0.00623, dev_loss = 0.07698, r = 0.9532, dev_r = 0.3973, (0.114 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 3020 - loss = 0.01426, dev_loss = 0.07696, r = 0.8653, dev_r = 0.4010, (0.077 sec/step)\n",
      "step 3040 - loss = 0.01586, dev_loss = 0.07353, r = 0.9002, dev_r = 0.4326, (0.087 sec/step)\n",
      "step 3060 - loss = 0.01473, dev_loss = 0.07464, r = 0.9092, dev_r = 0.4255, (0.085 sec/step)\n",
      "step 3080 - loss = 0.00934, dev_loss = 0.07497, r = 0.9097, dev_r = 0.4282, (0.108 sec/step)\n",
      "step 3100 - loss = 0.01143, dev_loss = 0.07281, r = 0.9175, dev_r = 0.4484, (0.100 sec/step)\n",
      "step 3120 - loss = 0.00914, dev_loss = 0.07304, r = 0.9397, dev_r = 0.4460, (0.106 sec/step)\n",
      "step 3140 - loss = 0.01294, dev_loss = 0.07492, r = 0.9086, dev_r = 0.4332, (0.087 sec/step)\n",
      "step 3160 - loss = 0.00651, dev_loss = 0.07296, r = 0.9396, dev_r = 0.4497, (0.101 sec/step)\n",
      "step 3180 - loss = 0.01130, dev_loss = 0.07409, r = 0.9163, dev_r = 0.4362, (0.092 sec/step)\n",
      "step 3200 - loss = 0.01073, dev_loss = 0.07401, r = 0.8814, dev_r = 0.4356, (0.091 sec/step)\n",
      "step 3220 - loss = 0.01171, dev_loss = 0.07583, r = 0.9289, dev_r = 0.4309, (0.082 sec/step)\n",
      "step 3240 - loss = 0.00707, dev_loss = 0.07400, r = 0.9479, dev_r = 0.4407, (0.100 sec/step)\n",
      "step 3260 - loss = 0.01026, dev_loss = 0.07335, r = 0.9175, dev_r = 0.4450, (0.089 sec/step)\n",
      "step 3280 - loss = 0.00533, dev_loss = 0.07299, r = 0.9614, dev_r = 0.4514, (0.082 sec/step)\n",
      "step 3300 - loss = 0.00621, dev_loss = 0.07596, r = 0.9532, dev_r = 0.4189, (0.087 sec/step)\n",
      "step 3320 - loss = 0.00434, dev_loss = 0.07337, r = 0.9598, dev_r = 0.4462, (0.110 sec/step)\n",
      "step 3340 - loss = 0.00686, dev_loss = 0.07470, r = 0.9504, dev_r = 0.4338, (0.096 sec/step)\n",
      "step 3360 - loss = 0.00464, dev_loss = 0.07333, r = 0.9636, dev_r = 0.4493, (0.086 sec/step)\n",
      "step 3380 - loss = 0.00999, dev_loss = 0.07374, r = 0.9282, dev_r = 0.4543, (0.097 sec/step)\n",
      "step 3400 - loss = 0.00458, dev_loss = 0.07328, r = 0.9683, dev_r = 0.4562, (0.082 sec/step)\n",
      "step 3420 - loss = 0.00498, dev_loss = 0.07194, r = 0.9543, dev_r = 0.4630, (0.084 sec/step)\n",
      "step 3440 - loss = 0.00525, dev_loss = 0.07262, r = 0.9517, dev_r = 0.4521, (0.078 sec/step)\n",
      "step 3460 - loss = 0.00467, dev_loss = 0.07247, r = 0.9688, dev_r = 0.4586, (0.106 sec/step)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 3480 - loss = 0.00220, dev_loss = 0.07173, r = 0.9796, dev_r = 0.4628, (0.089 sec/step)\n",
      "step 3500 - loss = 0.00624, dev_loss = 0.07225, r = 0.9576, dev_r = 0.4659, (0.079 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n",
      "step 3520 - loss = 0.00344, dev_loss = 0.07295, r = 0.9683, dev_r = 0.4580, (0.078 sec/step)\n",
      "step 3540 - loss = 0.00722, dev_loss = 0.07244, r = 0.9406, dev_r = 0.4595, (0.085 sec/step)\n",
      "step 3560 - loss = 0.00728, dev_loss = 0.07249, r = 0.9544, dev_r = 0.4605, (0.092 sec/step)\n",
      "step 3580 - loss = 0.00593, dev_loss = 0.07227, r = 0.9522, dev_r = 0.4618, (0.086 sec/step)\n",
      "step 3600 - loss = 0.00596, dev_loss = 0.07028, r = 0.9445, dev_r = 0.4804, (0.113 sec/step)\n",
      "step 3620 - loss = 0.00704, dev_loss = 0.07166, r = 0.9429, dev_r = 0.4623, (0.103 sec/step)\n",
      "step 3640 - loss = 0.00606, dev_loss = 0.07216, r = 0.9499, dev_r = 0.4571, (0.094 sec/step)\n",
      "step 3660 - loss = 0.00560, dev_loss = 0.07048, r = 0.9559, dev_r = 0.4754, (0.078 sec/step)\n",
      "step 3680 - loss = 0.00588, dev_loss = 0.07026, r = 0.9534, dev_r = 0.4773, (0.080 sec/step)\n",
      "step 3700 - loss = 0.00335, dev_loss = 0.07104, r = 0.9762, dev_r = 0.4675, (0.079 sec/step)\n",
      "step 3720 - loss = 0.00698, dev_loss = 0.07149, r = 0.9488, dev_r = 0.4754, (0.078 sec/step)\n",
      "step 3740 - loss = 0.00419, dev_loss = 0.07065, r = 0.9676, dev_r = 0.4743, (0.090 sec/step)\n",
      "step 3760 - loss = 0.00449, dev_loss = 0.07076, r = 0.9616, dev_r = 0.4783, (0.102 sec/step)\n",
      "step 3780 - loss = 0.00275, dev_loss = 0.07156, r = 0.9801, dev_r = 0.4729, (0.086 sec/step)\n",
      "step 3800 - loss = 0.00380, dev_loss = 0.07134, r = 0.9728, dev_r = 0.4751, (0.079 sec/step)\n",
      "step 3820 - loss = 0.00360, dev_loss = 0.07020, r = 0.9791, dev_r = 0.4864, (0.091 sec/step)\n",
      "step 3840 - loss = 0.00357, dev_loss = 0.07021, r = 0.9730, dev_r = 0.4789, (0.080 sec/step)\n",
      "step 3860 - loss = 0.00336, dev_loss = 0.07024, r = 0.9694, dev_r = 0.4818, (0.094 sec/step)\n",
      "step 3880 - loss = 0.00359, dev_loss = 0.07037, r = 0.9653, dev_r = 0.4856, (0.079 sec/step)\n",
      "step 3900 - loss = 0.00472, dev_loss = 0.07035, r = 0.9621, dev_r = 0.4869, (0.104 sec/step)\n",
      "step 3920 - loss = 0.00342, dev_loss = 0.07116, r = 0.9752, dev_r = 0.4781, (0.080 sec/step)\n",
      "step 3940 - loss = 0.00169, dev_loss = 0.07063, r = 0.9848, dev_r = 0.4840, (0.113 sec/step)\n",
      "step 3960 - loss = 0.00220, dev_loss = 0.07091, r = 0.9820, dev_r = 0.4825, (0.104 sec/step)\n",
      "step 3980 - loss = 0.00274, dev_loss = 0.07146, r = 0.9792, dev_r = 0.4765, (0.097 sec/step)\n",
      "Storing checkpoint to ./log_emoReg/ ... Done.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4VFX6wPHvm07ohCBIgIA0AelVERVFgg1cEXAtsKti77riqoiIu7jqT91V1wYWFMEuKoq4CArSMfQOAUInQGjpOb8/zp1kZjKTTDpx3s/z5OHOvefeOTMJ972nizEGpZRSKqSyM6CUUur0oAFBKaUUoAFBKaWUQwOCUkopQAOCUkophwYEpZRSgAYEpSqEiMSLiBGRMD/Hk0TkkorOl1LuNCAopZQCNCAoVWxi6f8d9Yejf9SqyhGRR0Vkt4gcF5GNInKxs7+aiLwnIkdEZJ2IPCIiyW7nGRFp6fb6PRGZ4GzXFZFvReSgc/63IhLnlnauiDwrIguAU0ALEaktIpNEZK+TnwkiEuqkDxWRF0TkkIhsAy4vxueLFJGXRWSP8/OyiEQ6x+o7eTsqIodF5FdXcPL3vSgVKJ/1mUqdrkSkDXA30MMYs0dE4oFQ5/BTwFnOT3Xg+2JcOgR4FxjmXG8y8CowxC3NjcAgYCMgwCfAAaCl837fAruAN4FbgSuALsBJ4PNi5OVxoDfQGTDA18ATwJPAQ0AyEOuk7Q2YIr4XpQKiJQRV1eQAkUA7EQk3xiQZY7Y6x4YBzxpjDhtjdgH/DvSixpgUY8znxphTxpjjwLPABV7J3jPGrDXGZAP1gMuA+40xJ40xB4CXgBFueXnZGLPLGHMY+GcxPuP1wHhjzAFjzEHgaWwwAsgCGgHNjDFZxphfjZ2QrLDvRamAaEBQVYoxZgtwPzAOOCAi00TkTOfwmdgndJcdgV5XRKJF5E0R2SEix4BfgDquKiCH+7WbAeHAXqf65ii2ZNCgtHlxznVPv8PZB/A8sAX4UUS2icgYKPJ7USogGhBUlWOMmWqM6Yu9KRvgOefQXqCJW9KmXqeeAqLdXjd0234IaAP0MsbUAvo5+8X9rd22dwEZQH1jTB3np5Yxpn2AeSnMHuxncz93D4Ax5rgx5iFjTAvgKuBBV1tBId+LUgHRgKCqFBFpIyL9nUbWdCANyHUOfwI85jQQxwH3eJ2eCPzZafBNwLNKqKZzraMiUg/bHuGXMWYv8CPwoojUEpEQETlLRFzX/AS4V0TiRKQuMKYYH/Nj4AkRiRWR+sBY4EPn818hIi1FRIBUbFVRbhHfi1IB0YCgqppIYCJwCNiHraJ5zDn2NLZ6ZTv2Zj3F69z7gCuBo9h6+q/cjr0MVHOuuwj4IYC83AREAOuAI8Bn2Pp9gLeBWcBKYAXwRYCfD2ACsAxYBax2zp/gHGsF/AScABYCrxtjfqbw70WpgIgukKP+qETkQuBDY0xcUWmVUlpCUEop5dCAoJRSCtAqI6WUUg4tISillAKq2NQV9evXN/Hx8ZWdDaWUqlKWL19+yBgTW1S6KhUQ4uPjWbZsWWVnQymlqhQRCWikvFYZKaWUAgIMCCKS4Eynu8U1d4rX8Qed6YZXicj/RKSZ27GRIrLZ+Rnptr+biKx2rvlvZ+SlUkqpSlJkQHAm93oNO+1vO+A6EWnnlex3oLsxpiN2tOa/nHNdUwD0AnoCTznD+AH+i50iuJXzk1DqT6OUUqrEAmlD6AlsMcZsAxCRacBg7HB9AJyh8y6LgBuc7YHAbGf6X0RkNpAgInOBWsaYRc7+D7Dzzhdn/nqllCpSVlYWycnJpKenV3ZWyl1UVBRxcXGEh4eX6PxAAkJjPKfxTcY+8ftzM/k3dl/nNnZ+kn3sL0BERgOjAZo2Lc6EkUopBcnJydSsWZP4+Hj+yDXTxhhSUlJITk6mefPmJbpGmTYqi8gNQHfsnO1lwhjzljGmuzGme2xskb2mlFLKQ3p6OjExMX/oYAAgIsTExJSqJBRIQNiN57zucc4+78xcgl367ypjTEYR5+52tgu9plJKlYU/ejBwKe3nDCQgLAVaiUhzEYnALhE4wysTXbCrRV3lLCXoMgu41Jmfvi5wKTDLmUv+mIj0dnoX3YRdN7ZcfPl7Mh8uKs6CVUopFXyKDAjO+rF3Y2/u64FPjDFrRWS8iFzlJHseqAF8KiKJIjLDOfcw8Aw2qCzFrhN72DnnTuAd7HKAWynHBuVvVu5l+tJdRSdUSqkydvToUV5//fVin3fZZZdx9OjRcsiRfwGNVDbGzARmeu0b67Z9SSHnTgYm+9i/DOgQcE5LISxEyMrRxaOUUhXPFRDuvPNOj/3Z2dmEhfm/Bc+cOdPvsfJSpaauKKnwsBAyNSAopSrBmDFj2Lp1K507dyY8PJyoqCjq1q3Lhg0b2LRpE0OGDGHXrl2kp6dz3333MXr0aCB/qp4TJ04waNAg+vbty2+//Ubjxo35+uuvqVatWpnnNSgCQkRoCNk5Os23UsHu6W/Wsm7PsTK9Zrsza/HUle39Hp84cSJr1qwhMTGRuXPncvnll7NmzZq8rqGTJ0+mXr16pKWl0aNHD6655hpiYmI8rrF582Y+/vhj3n77bYYNG8bnn3/ODTfc4OvtSiUoAoJWGSmlThc9e/b0GCfw73//my+//BKAXbt2sXnz5gIBoXnz5nTu3BmAbt26kZSUVC55C4qAEB4WQpaWEJQKeoU9yVeU6tWr523PnTuXn376iYULFxIdHc2FF17ocxxBZGRk3nZoaChpaWnlkregmO00XEsISqlKUrNmTY4fP+7zWGpqKnXr1iU6OpoNGzawaNGiCs6dp+AoIYSGkK0BQSlVCWJiYjjvvPPo0KED1apV44wzzsg7lpCQwBtvvMHZZ59NmzZt6N27dyXmNEgCQlioVhkppSrP1KlTfe6PjIzk++99D8FytRPUr1+fNWvW5O1/+OGHyzx/LkFRZRQRKmTm5GKMBgWllPInKAJCeKj9mDm5GhCUUsqfoAgIYU5A0GojpZTyLygCQnionQEwK1cblpVSyp8gCQhOCSFbA4JSSvkTVAEhW9sQlFLKr6AICGFOlVGmlhCUUqeBcePG8cILL1R2NgoIioAQkdeorAFBKaX8CYqAoFVGSqnK9uyzz9K6dWv69u3Lxo0bAdi6dSsJCQl069aN888/nw0bNpCamkqzZs3IdTrBnDx5kiZNmpCVlVXueQxopLKIJACvAKHAO8aYiV7H+wEvAx2BEcaYz5z9FwEvuSVt6xz/SkTeAy4AUp1jo4wxiaX4LH5plZFSCoDvx8C+1WV7zYbnwKCJhSZZvnw506ZNIzExkezsbLp27Uq3bt0YPXo0b7zxBq1atWLx4sXceeedzJkzh86dOzNv3jwuuugivv32WwYOHEh4eHjZ5tuHIgOCiIQCrwEDgGRgqYjMMMasc0u2ExgFeIypNsb8DHR2rlMPu1zmj25JHnEFj/IUoSUEpVQl+vXXX7n66quJjo4G4KqrriI9PZ3ffvuNa6+9Ni9dRkYGAMOHD2f69OlcdNFFTJs2rcBqa+UlkBJCT2CLMWYbgIhMAwYDeQHBGJPkHCvsEXwo8L0x5lSJc1tCrhKCtiEoFeSKeJKvSLm5udSpU4fExIIVI1dddRV///vfOXz4MMuXL6d///4VkqdA2hAaA+4r1Cc7+4prBPCx175nRWSViLwkIpG+ThKR0SKyTESWHTx4sARv6zYOQQOCUqoS9OvXj6+++oq0tDSOHz/ON998Q3R0NM2bN+fTTz8FwBjDypUrAahRowY9evTgvvvu44orriA0NLRC8lkhjcoi0gg4B5jltvsxbJtCD6Ae8Kivc40xbxljuhtjusfGxpbo/fNGKuvUFUqpStC1a1eGDx9Op06dGDRoED169ADgo48+YtKkSXTq1In27dvz9ddf550zfPhwPvzwQ4YPH15h+Qykymg30MTtdZyzrziGAV8aY/KayY0xe53NDBF5F6/2h7KkI5WVUpXt8ccf5/HHHy+w/4cffvCZfujQoRU+Q3MgJYSlQCsRaS4iEdiqnxnFfJ/r8KouckoNiIgAQ4A1Ps4rE/ndTjUgKKWUP0UGBGNMNnA3trpnPfCJMWatiIwXkasARKSHiCQD1wJvisha1/kiEo8tYczzuvRHIrIaWA3UByaU/uP45qoyytQqI6WU8iugcQjGmJnATK99Y922l2Krknydm4SPRmhjTMU0m+NWQtBGZaWCkjEGWxnxx1baKqagGKkcpr2MlApaUVFRpKSk/OFXTDTGkJKSQlRUVImvERRrKoeHaC8jpYJVXFwcycnJlLTbelUSFRVFXJzPypqABEVACHECQu4f/AlBKVVQeHg4zZs3r+xsVAlBUWUU6tQd6prKSinlX1AEBFcJQQOCUkr5FxQBIVSrjJRSqkhBERCceIAWEJRSyr8gCQhaZaSUUkUJioAQSg6RZJKrAUEppfwKioAQNu06pkeMJ0fbEJRSyq+gCAgSFkEk2VpCUEqpQgRFQCA0gnDJ1hKCUkoVIjgCQlgkkWSjUxkppZR/wREQQiOIIEvHISilVCGCIyCERRJBtnY7VUqpQgRHQAiNJEKyNCAopVQhAgoIIpIgIhtFZIuIjPFxvJ+IrBCRbBEZ6nUsR0QSnZ8Zbvubi8hi55rTneU5y0douFYZKaVUEYoMCCISCrwGDALaAdeJSDuvZDuBUcBUH5dIM8Z0dn6uctv/HPCSMaYlcAS4uQT5D0xYJOHkkKtrKiullF+BlBB6AluMMduMMZnANGCwewJjTJIxZhUQ0B1X7Fp2/YHPnF3vA0MCznVxhTqFj5yscnsLpZSq6gIJCI2BXW6vk/GxRnIhokRkmYgsEhHXTT8GOGqMyS7hNYsnLBKAkJzMcnsLpZSq6ipixbRmxpjdItICmCMiq4HUQE8WkdHAaICmTZuWLAehNiBIrgYEpZTyJ5ASwm6gidvrOGdfQIwxu51/twFzgS5AClBHRFwBye81jTFvGWO6G2O6x8bGBvq2nsJslVFITkbJzldKqSAQSEBYCrRyegVFACOAGUWcA4CI1BWRSGe7PnAesM4YY4CfAVePpJHA18XNfMCcNgQtISillH9FBgSnnv9uYBawHvjEGLNWRMaLyFUAItJDRJKBa4E3RWStc/rZwDIRWYkNABONMeucY48CD4rIFmybwqSy/GAeXAFBG5WVUsqvgNoQjDEzgZle+8a6bS/FVvt4n/cbcI6fa27D9mAqf06jcqjREoJSSvkTNCOVAUR7GSmllF/BERDyGpW1ykgppfwJjoAQqlVGSilVlCAJCLaEEKq9jJRSyq/gCAiuKqNcrTJSSil/giMgaJWRUkoVKTgCQphWGSmlVFGCIyC4SghaZaSUUn4FSUAIB0BMTiVnRCmlTl/BERBCQgGQ3OwiEiqlVPAKkoBgZ+gIMRoQlFLKnyAJCLbKKESrjJRSyq8gCQhaQlBKqaIESUBw2hC0hKCUUn4FR0AQIYdQrTJSSqlCBEdAAHIklFCtMlJKKb+CJiDkSpiWEJRSqhABBQQRSRCRjSKyRUTG+DjeT0RWiEi2iAx1299ZRBaKyFoRWSUiw92OvSci20Uk0fnpXDYfybdcQrVRWSmlClHkEpoiEgq8BgwAkoGlIjLDbW1kgJ3AKOBhr9NPATcZYzaLyJnAchGZZYw56hx/xBjzWWk/RCByRdsQlFKqMIGsqdwT2OKsgYyITAMGA3kBwRiT5BzLdT/RGLPJbXuPiBwAYoGjVLAcrTJSSqlCBVJl1BjY5fY62dlXLCLSE4gAtrrtftapSnpJRCL9nDdaRJaJyLKDBw8W923z5GqjslJKFapCGpVFpBEwBfiLMcZVingMaAv0AOoBj/o61xjzljGmuzGme2xsbInzkCthhKIlBKWU8ieQgLAbaOL2Os7ZFxARqQV8BzxujFnk2m+M2WusDOBdbNVUudESglJKFS6QgLAUaCUizUUkAhgBzAjk4k76L4EPvBuPnVIDIiLAEGBNcTJeXEYblZVSqlBFBgRjTDZwNzALWA98YoxZKyLjReQqABHpISLJwLXAmyKy1jl9GNAPGOWje+lHIrIaWA3UByaU6SfzYquMcotOqJRSQSqQXkYYY2YCM732jXXbXoqtSvI+70PgQz/X7F+snJaSCQnTKiOllCpE0IxUztFGZaWUKlTQBAQjoYRpQFBKKb+CKiCEaEBQSim/gicghIQTRg65uaays6KUUqel4AkITpVRjtGAoJRSvgRNQMgNCSOMXHK0hKCUUj4FTUAwIbaXUa6WEJRSyqegCQi4qoy0hKCUUj4FTUAwIeGEk02uDlZWSimfgiYgICGEiNFGZaWU8iNoAoIJCdMqI6WUKkTQBARC7OR2RksISinlU9AEBBMSSqiOQ1BKKb+CJiAgOg5BKaUKEzwBwTUOQXsZKaWUT0EUEEJsCUGrjJRSyqeAAoKIJIjIRhHZIiJjfBzvJyIrRCRbRIZ6HRspIpudn5Fu+7uJyGrnmv92ltIsP04JQauMlFLKtyIDgoiEAq8Bg4B2wHUi0s4r2U5gFDDV69x6wFNAL6An8JSI1HUO/xe4FWjl/CSU+FMEIiSMMMklV+uMlFLKp0BKCD2BLcaYbcaYTGAaMNg9gTEmyRizCgosWjwQmG2MOWyMOQLMBhJEpBFQyxizyNh+oB8AQ0r7YQoVYlcLPfO7m2D5++X6VkopVRUFEhAaA7vcXic7+wLh79zGznaR1xSR0SKyTESWHTx4MMC39cEJCDV2/Qzf3Fvy6yil1B/Uad+obIx5yxjT3RjTPTY2tsTXEScgKKWU8i2QgLAbaOL2Os7ZFwh/5+52tktyzRKRkFDPHbm6nKZSSrkLJCAsBVqJSHMRiQBGADMCvP4s4FIRqes0Jl8KzDLG7AWOiUhvp3fRTcDXJch/4EK9SggnS1H9pJRSf0BFBgRjTDZwN/bmvh74xBizVkTGi8hVACLSQ0SSgWuBN0VkrXPuYeAZbFBZCox39gHcCbwDbAG2At+X6Sfz5l1C0ICglFIeAqpYN8bMBGZ67Rvrtr0Uzyog93STgck+9i8DOhQns6VRoA3hxIGKemullKoSTvtG5bJivAPCyUOVkxGllDpNBU1AKFBCOKklBKWUchdEAcGtDSE0UquMlFLKS9AEBNxLCNVjtcpIKaW8BE9AcO92WiNWq4yUUspL0AQEKVBC0G6nSinlLkgDQgM4oQFBKaXcBU1A8GhDqOGUEHSxHFVaBzdCVnpl50KpMhE0AUFC3XoZVY+F3CxIP1p5GVJVX9oReK0nfHNfZedEqTIRPAHBu8oItNrIZcN3sHdlxb1fbg5/iMWt047Yf3csqNx8KFVGgicgePcygtOzYdkYe7N03WxKIicLju4qOh1AxgmY9mf4dFTJ38/d3lWw/D3fx5KXw2+vwqRLYcbd/q+xJxG+fcB+D1lpsP2XwALIqcPwYlvYNKvwdDlZRV/LZfuvsOpT38dcv6PQCN/HszNg1uOQnhr4+ylViYImIBQYhwC26+mW/8H+dZB+DFZOh+xM+3rnYs/zszPs2IWc7Px9R3bA4e2QdhTWfW1v5kd32RsTwJ7f4ZVOsHJa/jkHN9obSXYmJC+DjT/AtnmQeQrWfgX/jIMProIXWsOBDXB4mz1v7yqYOtymPbjJ7ss4Dh8Ng4+uta9Tk+GDIfD1XfByB5suOwMSp0LqbntTPZKUn5cFr8Bnf7Xbh7fBoc3w0zjYPNseO3EQju21n+nbB2HmI3Boi73m5tn2/ea/DD8+Yb8/sNUn39wHKVvh+H7Y+L29IebmwPd/gx8fh93LIPEj+PL2/M8HsPkn+P0jeHcQLJsMSb/a4PL+lTD//+xgwr0r4Z0B9roZJ+x1Xd/3vlVwfC98MtJ+t8bA8X3wy/MweyzMfwm+ugueqQ+7l9s0LtvmwcfX2fed8if7GQ9ugvevgC9ugdd62eDkzvW+oeHwywv5r11WTYeFr8J/unPaOHEQPv1L/u9LKTdiqlDDavfu3c2yZcuKTuhD8toFxH16GYYQ5OFN8EJL6PRnWDkVEGjaG3YuhAsehRVT4PgeOO9+aHYuHNwAv7wIGanQtA+cfaW9ceZker7JuffAsvdABJr3s1UJaUfsE2TfByHzhL1BRMfAGe3tk28g+j5ob+on9uXvq97AcyzFddNsvjd+l7+vYUcbxI7vsa/jz7c32WEfQFxP+L+2Xm8kgNffQ7W6UDfeBrfSiGkJKVt8H6vdFBp1hA3f+j8/JByiasMpPwMKm/SCXYt9HytMh2tsSdH7dxESbtuZ3A38J3S5waY/ttsG7O8fyT/ebggMc1uedfn7+avzjaugUsK2eZC8FPo97Pv4D3+HRa/Z7S43wODXKiZfqlKJyHJjTJFPJkETEPasX8SZ0weSK2GEPHnAPiWaQqohout73nwadc6/EbhE1bZPqJknPM+tVje/OiGup/03eUn+8fBoyDpV/A/R4xZY+o7drtXYMy9lpXm//Jujd9ApjpBwGwizTpZd3k53zc6Dv7hNCpw4Fb66w25XVEAYV7vg++Xm2L+3yJq2Cmvhq27pyyBfu5fb92jSs/TXUuUi0IAQNOtKhoSGA2BE7NoIoRGQnQ6974LYNraK5E9vwXcP2jrm236FDd/Y7UadoH5re6GFr4KEQIsLoWYjWx2SnQE1z7BVJx2ugbP6230rPoDWCVCnCSx+y15v4D/t+/7yAvS5C8Kr2fzk5sDeRDvP0oy74YqXbLXI4Nds3pr0hLZXQutBttQSEQ2TB8HO3yC2rS3FNO4OjbvCkrdg6LsQWcveCM6+ElJ3weyn7A1/5sOQmw1tLoONM0FCbamlyw1wyVO2mqhWI/t5962GbXOh+832nE9uhHPvtSWcjONQ7ywYX9emrd4A4s+DK1+xwdLl+H5bDXZwg+cvZcB46Hmbfco+vh82z4KuI+E8p9pp32rbE+yScXDGOVCzoa1ianaerRbLyYTacfb3EBoGcyYU/MWfMwyq1XF+H+/bQBXTEsIi4Nr34d+d89Pe9DV8MBhaDrAPC1v/Z/d3HG6rf4qyY4G94Z57r/178Ne2UBE2zYLWA+32u5fBrkXwxAHPYFCca8X3hYjqvo+/3d/+e/8a+7euqqygKSHs25pIwykXkB0SRdjY/fDzP2Dxm3D7fM8/YmPszdl7hbWKlJNl66WLknbU1mvXaGCDlIjN+/61tgqmsOuDfY/EqRDTCpr0KHl+t82D6vVtkPAnN9eWuFZ/agNJ6wQ4o0P+95yVBkvehh43e954Dm2xVVaB/D6O7LA38mO77Q28953Q4Gx7LCfbBmKAyBr552Sl26q4lC3Q8hKbzxCnaW3nYph8Kdw6x7a9SAjs+A02zIRjyYXnZVyqbRP6dGT+67J2cKPt9tr3ARs0047Ac/H5xx/ZaqsMX+9lX//le9s+4y66Poz6zrbVdP6zffDJzbKfd8dv9gHi9d6en8ubq1TSOgEG/gP+0xVGz4Uzu/jP+8y/wZI3K67kFOS0ysjL/u1rOeP9c8kKjSb8yb12Z0525d741enPGBto3SUvh3f6F37euFTP6hnXjS83F5ZNgs7X21IewJrPbeP+I9ugekzBayXNhwbtYMtP8PsUuGmGzdPEpvk9mO5ZAR+PgEOb8s+r2cg2spelR3fYEpfHZ3UCwpldodMI23mgxy1w+Yu24X/R69D7DltlBbBrKUy6xG5f9gL0vLXg+xzeDsvfhYvH5QdoVWJlWmUkIgnAK0Ao8I4xZqLX8UjgA6AbkAIMN8Ykicj1gFurGx2BrsaYRBGZCzQCXF03LjXGlNuMcyHOjd+I2x+XBgNVFO9gAPbJ98p/5zcY+7Jyumf1zMpptqRz8qCtsjuSBAOftccW/df+u3s5tL7U8zrpx+C9yz33pWyF+i09u7Ou+dwzGID/YHD1m7a0VJIBddvn2byfe68tjR136+iwZ4X9cX2W/etsNejPz9rS1a8vQrV6nqWrmQ/bn7/v8SwZfn6zvUbHEXBGu/z9aUdsCTPjBCx+wwaUsEqsmvuDKbKEICKhwCZgAJCMXRv5OmPMOrc0dwIdjTG3i8gI4GpjzHCv65wDfGWMOct5PRd42FlKMyClKSEc2r2V+m93JTOsJhFPFFHcVyoQqz61XVKLY/Dr8PWddvvJFNi/Bt66IP/4uFRbitj0ve3Rtn5GwRv3hY9Bt1HwYpuS5Xtcqu2K66vNpbLUaQr3rbJddzd9Dz88BhlO19g+d9v2ppBQePMCG2RqNLRVfaPnelZN7Vxke9td9R8tWbgpyxJCT2CLMWabc+FpwGBgnVuawcA4Z/sz4FUREeMZba4D3DrkVyxTzTZ87q7fl+bl/F4nM7LZcuAEnZrUKTqxqro6XgtznoGjOwI/xxUMwN7QvbvR/m+8fZJ2qe2jkXbuP+2PL9dNsx0ePhlpG+kBetwKcT1sl+Nm59p9ba/0HxCi6lT8tC5Hd8LTfv6/LHwVDqyDOs1sMID8Ltg//8OWeKLrwapP4Aun+qnnLXbszYH1tguuybUBxd32X6BJby1huAmkhDAUSDDG3OK8vhHoZYy52y3NGidNsvN6q5PmkFuarcBgY8wa5/VcIAbIAT4HJhgfmRGR0cBogKZNm3bbsaMY//ncpJzIYOizU/jrZedz4/klfLIK0E2Tl/DLpoOsfXog1SNLVy2Vm2v4cPEOhnVvQlR4aNEnBGDlrqNk5eTSPb5emVyvsuTmGoa8voC7L2rJpe0bVk4m/tPNNkg3PMf2iqpsT6bkV4X6av/wtm81vNE3/3XH4dD/CXj5HGg3GAY9b8eHnDwEc//h+xoN2sOBtYHlr9soz5HsPW6FpW8Hdm5hLhgD8yb6Pla3ORzZbrc7joBTKXag6qbvbenj0gm2s0PrgbaX3//GQ587bZpTh+2gxn6PQNNepc9nJTmtup2KSC/glCsYOK43xuwWkZrYgHAjth3CgzHmLeAtsFVGJc1DaIiw3TQiO6T0TwPGGJKPpNGkXrTP44k77RiE7JzSN9jPWLmHsV+vZf+xdB4Z6D2QrGQGv2bn3kmaeHkRKU9vJzOzWZWcygPTE1k7PqFyMuEayzL0XXjV+f8WWdvelE+lVFw+zux+MZqhAAAgAElEQVRi69Pd28WKCgZgA9m4VBs83J+iH99ve6GFhNqeX1AwIMT1tONrWlwI17xtez21v9oO8Ixta5/awd5U/9XcllKufMX+uCQtsAGh7RWFD0x0F1ETulxv2xBc/AUDyA8GAKu8KimWvA1rvrCDNzsMtd22F72WP3jP5dAmuH+VHaB5dCe0udz53RsIi/RMawyc2G9H1bu6/gYiO8N27Xa1paQdsW1IdZsFfo1SCiQg7Abcy61xzj5faZJFJAyojW1cdhkBfOx+gjFmt/PvcRGZiq2aKhAQyoo4/zlyy6BT1SfLdvHo56v57PY+hT5lG69Rv/M3H6JJvWo0i/HTn9uH4xl2qowjp4ox/06QcH27IYHc+AKQmZ3LwRMZNK5TLfCTml9gx0ZEx8D1n9ub2pUv5/ccanWp7YoZEuY55qEovkZK+xJVB4Z/CM3PD/zavojY8Sgu4VEF0zy+3zYOn0qxN/vQCNtY3n6IHU/j6nbsqpZyia4Hf9tu03iLP88ei65npyjZuchzBoCLx9ondpfbF0DDDnb76E7bG+nges9rXjIOwqJg3r+g5cX26d+fnIz8kfxrPrM/vhzdAV/e4cxs4CY0EsbstN/Xxh9skIquZ3//YHt/gR2TtOBlu33fStvBAGzwyDgOM+6BdV9BWDW4axGcTPHsydb7TtuOEkh39FIIJCAsBVqJSHPsjX8E8GevNDOAkcBCYCgwx1X9IyIhwDAg7y/WCRp1jDGHRCQcuAL4qZSfpVAhzj2jqCqy5TsOc0atKOLq+n76B1ixw9avbj5wgu7x9cjMzuXej3/n4YGtadmgpt/zbphkp1bwfjI3xrBm9zHOiatd4BzJS1Pwejm5hhDJD3bB5Mvfk4mOsH++ZfXxH/tiNZ+vSM6r6lux8wirdh1l1HmFtDoN+pedsiS6HrS6xP6AHaB4Rgc7xsPVuPnUUd/15A9vzp9KO6IG3PKTHeT3fAt7vN0QGPQcTL/Rc8T737bbAYDedePlxRUkXIMWATpfF9i50YVUT7qOjfwmf9+pw3ZEf3iUHUBZr0XBJ/HrPs5PO+VqO+Aw4Z92XA7Yrq5gb6Yz7rEN+LWbQupOu7/nbbbXVNYpG1xcuo2ypZmdC20vqvMfhncTCgYDsAHl2TPsgMoV7xc8/p+uBfe90gn+OsuOB5p2vZ0SxyU7zR73tuh126W3kY9jZajIgGCMyRaRu4FZ2G6nk40xa0VkPLDMGDMDmARMEZEtwGFs0HDpB+xyNUo7IoFZTjAIxQaDMqhI9C8kr4RQeEC45r8LgcKrUyQvuNh/f995hB/W7iPlZAaf3p7/dCQEdqeatnQXj32xmsmjutO/7Rk+38vb/mPp9PrH/3hmcHtu7BMf0PuUteycXMJC7c3ul00HOaNWFG0a+g+IZemB6fnTdbsHxJ83HODcljFEhgV+kzx0IsOeu9H2ep69bj+fr0jm1822CaywgLBmfxo1IhsS7+tgrFdblYjtn9+gnR0RD/bpvkYD+zPqO3vjq3Wm8yHX2ZKH60Z8y2yY+5ytumlzeeE32arO/bO5BhcWlva2ef6PN+4KdyzIb1PJzSkYRDNP2hu0+xiLLjfkb//p7fwG61vm2FJJ4sewY77d5ysYeHPNKAAw2U9VUtNz7ewDLq4G/sbd7dxk5SygNgRjzExgpte+sW7b6cC1fs6dC/T22ncSO2ahwrgCwj9mbuCtX7ax7IkBAOxLTadh7SgysnN4d0FSgfPSMnP4KnE3I3o0QUT4ecMBViXbiO6qEnJVQ5X0SX3jvuMAJB0qbH4jz0D23Pf2D+vJr9dyY5941uxOZfrSXYwf3L7MSww/rt1HpyZ1OKNWfjXC7qNpnDdxDv8a2pFh3Ztw02T75Oqr9HPgeIbHuYXZuO84dauH89O6A1zXs0lAn8WVJHHXUf7y3lJu6tOM8YM7FEg3e91+Dp3I4LqeTQF4d8F2nv4mv7Ncveq2fen+6YkB5RXgiv/YG0LA7TGXOz2Iaja04wnOvjL/WHxfz7S1Gxc8/8JHodUAiDkr4Dwqh+sPxVeJyt+0HC4dh9lqnt3LIa6b/elyg+3FtHOhna4d7PQd1WPzq+DmPWdLf5dOsIF9+Xt2Rt/j+/JLKmAb83vdBo272ZmQQ0Jte4KrVBRIB4EyEDQjs9y/y0MnbB3lL5sOctPkJbx5Yzc27TvOi7M3FThv4vfreX/hDhrWiuKitg34y3tL8465SgiuaqiQUv6+XLf87JxcHvxkJauSj9Ixzj6xbD14ktxcQ4jzJkt3eE617Lox3X9JK2JqeBWtA7Bi5xE6xdVh84HjzF67n3submXzZAyjpyynab1ofvnbRXnptxywE/p9s3IPw7r7n7/G1d4y4+7z8j6Luzkb9nNWbA0a16nG3tR0Br6cP+vo2Y1q0qVp3SLz7vraU9Nsnfv2Q74n1Lv1AzuGxRUQnvl2ncdxf9WJw95YyCe39ykyH8XSthQN+o19VEOo8tekZ8EJ/BqcbX+anWcHAnrP5dT/cc/X3UbZH5fsTFsCcFVzQX43WPfAVUHVwkETEEJ93K1X77ZP+om7jvpdXvmgU51wMjO7wDHj9a934+ax9CxqR9tGoNXJ+fWEFz7/M88M6cD5rWLJzsnl912efb437DvOjJW2oSspxZYalmw/zH/mbOG+S+yNetfh/Ln5XTdCKH4pJT0rh/V7j/Gn13/j3v4teevXbaRn5XLHhWcRFhrCCadRe+fhUx4BaUbinoCuv2ibDVyfLkv2GRD++p7/gYbpWYGtqub63t/51dZKev8ul+84wsHjJV/3eEnS4aITqeAW26ZgFWEgwiI8g0ElC5qhfN436/gx3/HCjxsB+4SZlVPw5pNyIoOZq+0AmJW7Cg7UefKrNew6fCqvXSLEqyfTX53SxLxNB7ny1fl55yWlnOLOD1eQfOQUL/20Ke/aAny3ai8rdvpeLW3ZDt83pk5P/5i3feRUJgu2FFwz4JdNB/NumC4rdx2l7ZM/8NFiW3T9dtVeMrI9v4cbJ+U3Yr44eyPpWTk88+06Pl+RP9r7hVkb87bTMnN85nHKoh289vMWn9+zP969tPxJOZnJweMZeXX+870+/zX//Y3bP1yR99rVZqCU8hREAaHgPteT5OQF2zmZ4VkC2Jeazhcr8nvXvv3rdm6fsrzANa5+fUHeuVsP2moU11P15gMnmLFyDyMnLylw3vGMbPo+9zNrduevXHXgeAZ3TV3B2K99D/JJOZFZ5M3s4hfncf07i5m78QDxY77jtZ/tojQ3TV7ChO/Wc8DtSdk1HuGz5fbmvu3QybzvJMcYpixMItEtEH6zci/PfreeSfPd+nUDr/6cv/BN73/+j72paaRn2cDgXg3z/KyNTFm4g8XbUvh+9V4OHi/+jXnX4VM+g86AlzwbFV+dszkvD966T/iJ5COnAgw3BW05cIL4Md/x6+bTcAlWpUohaGY7BVsq8Kd+jcgyeXLc/s/LaP7YzKITOs5pXDuv6ipQfVvWL/AU7K1PixgWbrNDQT6/o09e76lAPZrQlud+2FBkul7N67F4u++Sy6pxl9Jx3I8e+x4Z2Ibn3UoUhXn26g58uGgn747qQURYCG/M28pbv2yjVYMabD5wougLFGJ0vxa8/es2j+qlOtHhHPUz3uNPXRvz5OXtqFs9ghd/3Mh/5niu/rb9n5fxdeIeEjo0LLMR5UqVldNqpHJVUFbVCK6bcKCKGwygYJVIUfkobjAAWLc3sDV3/QUDgHE+SjqBBgOAKQt3sGHfcd5fmMR/527N21/aYAC2Tcb7WaiwkeVfrNjNrsOn+KufLqg/bzzA/dMTuWmn7x5OSlUFQVNlVFHmbvxjVCN8szKwRuPCJKWUbvnMDU53XPdgUFYSfbQJncgo2HHA3dKkI9zx0Qqfx+ZssGMY9qaWvPFaqcqmAaGMvfXLtqITBYkVOyt4xswK4l1dBPmDEKtQDaxSBWhAUKpMaURQVZcGBKXKgPd0JkpVRRoQlCoD7oMDlaqqNCAoVQa+dkZuL9l+mPgx37H/mDYuq6pHA4JSZci1fkVh3XGVOl1pQFCqHFSlAZ9KuWhAUKocHT6ZyfF0bV9QVYMGBKXKgauA0PWZ2fT555zKzYxSAQooIIhIgohsFJEtIjLGx/FIEZnuHF8sIvHO/ngRSRORROfnDbdzuonIauecf0swrgOp/rBe/mkT5//LBoKiRkArdbooMiCISCjwGjAIaAdcJyLtvJLdDBwxxrQEXgKeczu21RjT2fm53W3/f4FbgVbOT0LJP4ZSp5eklFMea1YoVRUEUkLoCWwxxmwzxmQC04DBXmkGA65FRT8DLi7siV9EGgG1jDGLjG19+wAYUuzcVyHvjupR5tcs7QptSinlLpCA0BjY5fY62dnnM40xJhtIBWKcY81F5HcRmSci57ulT3Y739c1ARCR0SKyTESWHTxYuonjkiZezsqxlxIRWvFNJxe1LXpVpAcuaV2sa2775+W8PLyz3+OXndOwYD7axPpNHxlWcd9LtTKYIrp+CZYK9WXOQxeUyXWUqurK+w6wF2hqjOkCPAhMFZFaxbmAMeYtY0x3Y0z32Fj/N7NA1Y4OZ+OEBB5NaOs3zfjB7fniznOZNDJ/+vB/De1YIN300b19nt+4TjVm3nu+z2MAT1/Vnlv6FpxG+b5LWlEzsuCM5MN9rFncqkENAIZ08RlHAehzVn2P1/de3IqXR3Txm37mfecT4yw0X1KxNf3fpBvXqcbNzueODC/4p3ez13fSI973espDu8UBcEHr/L+HLk0LLs8ZiA6Na9EitkaR6d66sVuJru9y5GQmUxYmaXdUdVoLJCDsBtzvSHHOPp9pRCQMqA2kGGMyjDEpAMaY5cBWoLWTPq6Ia5YbESE8tGB9y5WdzmTThEHc1Ceerk3rcvHZZ+Q9yV7o3HzCnHqa63s1pVeLmALXAFgwpj/tzvQf90TgiSva8fuTAwocG9DujAL7Inw8ubsvCfr0Ve15ZGDh67le3aUxDw5oTe1q4fz3+q7ceeFZADSpVy0vzVmxNVj+5ACPYDX1ll4kTbyc2Q/04/2/9uQO5zyAdo1qMe+RCz3e57yzfH8nLWKrs2BMf26/wJ4f6qNG8e+Xne3x+tPbz/V4PerceD67vQ+9mtcD4M+9mrLl2UHc0LsprwzvwphB/oO8P65ZSj+6pZffNCN6NCHMx99Lcdw/PZEnv17L+r3HS3UdpcpTIAFhKdBKRJqLSAQwApjhlWYGMNLZHgrMMcYYEYl1GqURkRbYxuNtxpi9wDER6e20NdwEfF0Gnydg1/dqxk19mnnsCwuRAjffXx+9iLkPX5j3um71CH7920WMu6q9z+tOcys1eJcSru/VFLAByXWtpY9fUmReLzunEQA/PtAvb9+t/VrkbY88N567LmqZ9/q2fi24t3/+a4Ae8fXytged04i+LW3poXGdanjr28oe+/yOcznXSdfqjJpc0DqWRxPactsFLXj9+q7MvO98msVU59t7+nLbBS08ruH9PSa0t9VXrifkEB8NIKGFNIokTbyccVe1p3t8PYZ2i2PBmP50a1aXsNAQJgw5h6Yx0dzUpxnXdI3zew2AmlG2BPaYEzwedgJpi9jqAHR1K2k8M7g9U2/pxcRrOnJ+q1hu7N2Mz+/oA9hSSsNaUYW+l7t5m2x1Z66WENRprMiA4LQJ3A3MAtYDnxhj1orIeBG5ykk2CYgRkS3YqiFX19R+wCoRScQ2Nt9ujHGN6b8TeAfYgi05fF9Gnykg1SJCA1rZqn6NSOLrV897bQw0qRdNuI92iP89dAG93UoN7c6sRe1q4XmvXTe8MLcbn3cVy6XtbQlh1v325h8eKvQ5K4akiZfT+oyaeelc1Sa+3H7BWTx4aX6JoWm9aK7r6VntFFc3GoB+rQtWw13YpgEbnkmgWzPfVTaPDTo7L0gBdGhcm3Ma1wbs9/rdvX2Z/7eLPM5xlZhct8PqEb7bELyr8vq3bcCw7p6fVUR8BrLoiDBeHNYp7/Uzg9vz/NCOeXkDeOrK9kSGhfDXvs1Jmnh5XrVTo9rVeH5oR9680VYTnntWDDf2ic8LiOGhITwzpAPdmtXj8zv6MGFIh0IDmD++/m5cHvl0Je8t2O73uFLlLaAlNI0xM4GZXvvGum2nA9f6OO9z4HM/11wGVPpag1HhIaRn5QLQLCbaf8IA/u/XigovsG/2A/1IPmq7Hz40oA0hIvypq2e9/6SR3fMCR0KHRmx+dhDhoSH89/qunN2oWE0uQH79fOc4+7T7xOVn493pq2lMNEv+fjH1a0Tyrx8KLmtZ3HWBE9o35N6LW3HL+c19fg9XdDwTgAY1I3loQGsGd25Mv+d/5oxakfx4/wXkOE/Od1x4lsdazpNL2DurVlQYN/aJB+Da7k3y1tMe2i3ObzC91mmrWTn2Uqr5CVgA3ZrZ0taoc+N5dub6YuWrsCDy6fJkPl0Oo/ws06lUeQv6NZWXPTGAnFzDyl1HOddP/bcn30X+uQ9f6LNBtUGtKBo4VQu1o8N9VjVdfLZnu4HrKXKQ21N4IFaPu5RN+08QHWF/refE1WbDMwl+b+4NilHlUZSw0BAeHOC7l9Q9btVXIsI9F7cCbBVYbI1IakcXDCCl8dVd53Fm7ZJ/tkDzc2u/Ftzar0VesAmEr7YrpU4XQR8Qaji9enxVnbgTP0WEnx++kJQTGR7VSuXpmcHtifRzg68ZFV6gmieQJ/2HBrSmSb1CSkcldHWXxnz5+24eutR3g7d7FVhZ6tykZD2OSqp3i3os2hbY7KZTFu5gzKC2hFVC12elihL0ASFQ1SPtjfVyr6f25vWr07yCggGQVw1SllxP7GXtxWs7MfGac4p93rDucYSGlO0Ns2ZUGMfTy2cKiccGnc3dH68IaGTyO/O3U7d6hEcnAKVOF1KV+kV3797dLFu2rNLe/8jJTGpVCy9RY6KqXDtSTrJ8xxH+VEQvpNIoTtXRrPv70Swm2qME5zo/aeLlZZ43FdxEZLkxpntR6bSEUAx1SzloS1WeZjHVaRZTviW5tU8PpP1TswJKO/DlX0ho35AaUWHEx0Rzd//yKaUpVRxakalUGanuY5R5YX5Yu4/Plifzwo+b/KZZt+cYny1PJuVEBlMX7yQn17NEP3XxTh75dGWJ8quUNy0hKFWGWjWoweYDJ4p93m1TPKtCJ3y7jnfmFxyTkJ2by01u7Uh//3I1AM9f26lAWqWKS0sISpWhj27tVaJ5j2at3e/x2lcwALsCm1LlRQOCUmWoQc0oLm1fcJbZ4li4NSWgdN+u2uNz/9eJu1mdnFqqPKjgpFVGSp1mrnt7kd9jHy7ayd0XtSQsNIT5mw8VOO7e00l7K6ni0hKCUlXIoRMZ3DZlOfFjvmPd3mN5+5/9bh27Dp/ySDt73X6dblsVi45DUKocFGdMQnn619CODPOxnoYKLoGOQ9ASglLlYNb9/ejdol7RCcvZ1gMnSE3L4puVvtsblHKnAUGpctCmYU3aNiz+TLVlLTREuG/a79zz8e9sP3TS49jq5FQu+b95fLxkZyXlTp1utFFZqXJy78WtOHIqk68TK+/pPCxEmLvRLs6TmW2nef902S4e+WxVXprHvljNdT2bVkr+1OlFSwhKlZN61SN4ZUQXLjm74LKoFcV9ksCBL//CR4t38N95WystP+r0pgFBqXJWkoFqZeWlnzynxfjXDxt9Lunh6lzy3aq9Psc3vP9bEte/4787rPpjCCggiEiCiGwUkS0iMsbH8UgRme4cXywi8c7+ASKyXERWO//2dztnrnPNROenQVl9KKVOJyLwp66NmXpLr8rOCqlpWWzzaksAuH96IgB3TV3B3VN/L3D8qRlrWbAlsAFzquoqMiCISCjwGjAIaAdcJyLtvJLdDBwxxrQEXgKec/YfAq40xpwDjASmeJ13vTGms/NzoBSfQ6nTlojwf8M6563PfDoqTTtHWmYOV/5nPquSj5ZhjlRlCKSE0BPYYozZZozJBKYBg73SDAbed7Y/Ay4WETHG/G6Mcf2lrQWqiUjBdSaVChL/ua5LZWfBr4fdZk1dmnSY6UsD6320Mvkoq3enMuG74q0vrU4/gfQyagzscnudDHiXffPSGGOyRSQViMGWEFyuAVYYYzLc9r0rIjnA58AE42OUnIiMBkYDNG2qPSFU1XZlpzO5stOZAJzMyGbIawtKNDtqefhseXLe9rVvLARgeA/9PxdMKqRRWUTaY6uRbnPbfb1TlXS+83Ojr3ONMW8ZY7obY7rHxha+7rFSVUn1yDDuu+QPtDBO1Zn0QPkRSEDYDbiPfY9z9vlMIyJhQG0gxXkdB3wJ3GSMyevvZozZ7fx7HJiKrZpSKqhc0fFMVo69tLKzEbD0rJwyv2Z2Tm6BhX9U5QgkICwFWolIcxGJAEYAM7zSzMA2GgMMBeYYY4yI1AG+A8YYYxa4EotImIjUd7bDgSuANaX7KEpVTbWjwxl7hXc/jXw94ytvCoz4Md+xYV/+JHptn/yBkZOXkJtr+H3nEf70+oK8AW9Lkg4XOP+8iXMY8dZCv9fffTSNlo9/T8LLv5R95lWxFRkQjDHZwN3ALGA98IkxZq2IjBeRq5xkk4AYEdkCPAi4uqbeDbQExnp1L40EZonIKiARW8J4uyw/mFJVyV/7Nvc5XfX8Ry/i4rMrt0d2wsu/eryet+kgz3y3jie+WsOKnUe5afKSAufsTU0D7A1/0baCgcLllvftZJWnSztKsAto6gpjzExgpte+sW7b6cC1Ps6bAEzwc9nKG62j1GluwpAO9IivR1zdaG49vwW9WsTQuUkd7vn4d7YcOMH6vcdoWCuKnx66gC0HTjDktQVFX7QMvbsgyef+hVtT+HTZLr74fTcvDc9f1nPepoNc0Nq2Ad44aTEd42rzyMC2HE/PqojsqgDpSGWlTiM1o8K4uktjbujdjDYNawIQEiJ0blIHsN1WJ420sxiLQI3IsLxjLpXZtfW6txfxxe+2ifGB6fndWEdOXsK+1HQAft18iNd+ts2Judp2cFrRgKDUaWT1uIG8NLxzoWlqVQsH4OoujfP2/eW8eACqR4RyZacz+f3JAQXOu+PCs8ouoyVwMjPb4/XCrSnscYKEt9xcQ06uwRiji/xUIJ3tVKkqpkZkGOvHJxAZlv8898CA1h7VOHWrR3ic07JBDR5NaMt/51bexHYPTE9k0/7jea8LWyr0ghd+JuVEJpnZubRpWJPv7j2/IrKYJyM7h9xcqBYRWqHvW9m0hKBUFVQtIpSQEMl7LW77XYZ1j8vbDpX8tO7ur8BxEKuSU0nPyvV7fO7G/Nlrdh1O41RmDtm5hrV7jvk9p7wMfOkXzh77Q4W/b2XTEoJSfwA1o8J5ZGAbBrZvmLfvySvaESLCtKW7GNjeTsH9wrWd8qao+PaevjSLieblnzZXSp69jXp3KQB3X9SywLG9qWmkZebQIrYGANe/s4ie8TGs2HmE1btTWfHkAKYu3smq5KNMvKZjqfOSlHKq6ER/QLqmslJ/cIdPZlKnWnheiWJvahrr9x6jf1sbJHo++xMHjmcUdonTxvrxCYSHCi0f/95jf9LEyz3Wsb7k7Aa8M7KHz2tk5+QSFlp45YjrWr66AldFuqayUgqwC/W4Vy81ql0tLxgAzH+0P389rznLn7iExwa1rYwsBuzssT8UCAa+/LT+AJvd2itcNu8/TsvHv+exL1azM0hLAYXRgKBUkIsIC2Hsle2IqRFJa6era1WzI6XgGg8DXvqlQA8lV3vEx0t20u/5nwG7ONAnS3eRlllwWo6cXMPWgwUHze0/lu7R5uGefnVyaok+w+lAA4JSKk8dp0vrn3s15f2/9qS28xpg7dMDKytbRbrg+bk+97/yv81Mmr+dnSmneHXOZg6dKFg1Nn/LIf72+SqenbmuwLFR7y7h4hfnFQg4Q15bwKh3l/LV77vJzslvKH9p9iaufHU+a/fkB4XUtKwqM95CG5WVUnm6NK3LGzd05cI2DYgKD2XlU5dy46TFXNnxTKpHhnFFx0Z8u2pvgfPanFGTjT6qaCqbq8H8mW8L3uzBjoV48is7jdrPGw5y+GQm9dy67P662c7gf+hEJntT07l/WiJ3XnQWe53xE/dPT2T/sXRuu8CO8VjpLBJ06ESm828G3Sf8xAOXtPY5s216Vg6rd6dyLC2LzOxcBp3TqCw+dolpo7JSKmDGGDKyc2n7ZH6XzNkP9KN+jUge+nQlczb4X/jwrovOyhuhXNUM6x7HJ8uS/R5/aEBr7rm4Fde/s4gFW1KYcnNPejavx+b9J7jiP/MBWDd+INERns/gg1+dz0q3KqbyasTWRmWlVJkTEcLcGqg3Tkig1Rk1qVs9gsmjejB5lP97ziMD2zKk85kVkc0yV1gwAHhx9iaMMXnrTt84aQltnvghb6EhgHZjZxU4b6VXe8O1b/zGx0t2kptr+Gndfka8tZApC5NKnf9AaQlBKVUsxhiaPzaTey9uxYMDWvs9DnBL3+Zs2Hec+VsOkTTxcg4cT6fns/+r6CyfVrb94zJmr9/PbVOWB3xO4tgB1ImOKDqhH4GWEDQgKKXKXJfxP3LkVJbPKpDVyanc/P5Sj7EPS/5+MQNe+oXUNJ391J+Fj/WnUe1qJTo30ICgjcpKqTI3877z2X6oYFdQgHPiarPk8UtYu8dOZdExrjbhoSGM7NOMf8/ZUqz3qRMdztFTwRFETqRn27Uoy5EGBKVUmWtUu1qRT7Ptz/S8uz0woDX3XdKaxdtT2Jlyird/3cbWgzaofHtPX/amplMtPJQbJi0GYO7DFxJfvzpHTmZyNC2LfanpHD6ZyV1TV1C/RiQThnTg9g8Dr5Y53fmZjqps3yOQKiMRSQBeAUKBd4wxE72ORwIfYBe9SQGGG2OSnGOPATcDOcC9xphZgVzTF60yUip4GGOYNH8713SN85i9NT0rh3V7j9G1aVjip1YAAAlSSURBVF2/54nX3TMzO5ctB04wc/VeRvRsQs2ocDo9/SMA9WtEeoxPuKF3Uz5ctNPj/EcT2vLcDxvK6qOVSEVUGRUZEEQkFNgEDACSsWssX2eMWeeW5k6gozHmdhEZAVxtjBkuIu2Aj4GewJnAT4CrFarQa/qiAUEpVZbW7E6lZYMafLAwiciwUG7q0wwRIeVEBst3HKF2tXB6tYjJS5+bawgJEZIOneTCF+YSERaSt6Z0j/i6LE06AsAbN3SlVrVw/vy2Lc00rlON3UfTSpXX0nRJLcs2hJ7AFmPMNufC04DBgPvNezAwztn+DHhVbIgeDEwzxmQA2501l3s66Yq6plJKlasOjW211eh+nosHxdSI5FK3mWNdXHNCxdevnneDTj2VBYLHqG6XV0Z0plHtavRsXg+AtMwcosJD+DpxD03qVeOcxnU4mpbJnqPpLN1+mIvaxmIM1IgK4+DxDNbvPcbny3fTr3X9Mv3c/gQSEBoDu9xeJwO9/KUxxmSLSCoQ4+xf5HWua5mnoq4JgIiMBkYDNG3aNIDsKqVUxakdXTAQuAzu3NjjtWu9iiFuq901qBlFg5pRBZZCbVS7Gh3j6jC8R8Xd9077gWnGmLeMMd2NMd1jY2MrOztKKfWHFUhA2A00cXsd5+zzmUZEwrCdo1IKOTeQayqllKpAgQSEpUArEWkuIhHACGCGV5oZwEhneygwx9jW6hnACBGJFJHmQCtgSYDXVEopVYGKbENw2gTuBmZhu4hONsasFZHxwDJjzAxgEjDFaTQ+jL3B46T7BNtYnA3cZYzJAfB1zbL/eEoppQKlU1copdQfnM52qpRSqlg0ICillAI0ICillHJUqTYEETkI7Cjh6fWBQ2WYnbKi+SoezVfxaL6K53TNF5Qub82MMUUO5KpSAaE0RGRZII0qFU3zVTyar+LRfBXP6ZovqJi8aZWRUkopQAOCUkopRzAFhLcqOwN+aL6KR/NVPJqv4jld8wUVkLegaUNQSilVuGAqISillCqEBgSllFJAkAQEEUkQkY0iskVExlTweyeJyGoRSRSRZc6+eiIyW0Q2O//WdfaLiPzbyecqEelaxnmZLCIHRGSN275i50VERjrpN4vISF/vVQb5Giciu53vLVFELnM79piTr40iMtBtf5n9nkWkiYj8LCLrRGStiNzn7K/U76uQfFXq9+VcL0pElojISidvTzv7m4vIYud9pjszHCN2FuTpzv7FIhJfVJ7LOF/vich2t++ss7O/Iv/2Q0XkdxH51nldqd8Vxpg/9A92NtWtQAsgAlgJtKvA908C6nvt+xcwxtkeAzznbF8GfA8I0BtYXMZ56Qd0BdaUNC9APWCb829dZ7tuOeRrHPCwj7TtnN9hJNDc+d2GlvXvGWgEdHW2a2LXAG9X2d9XIfmq1O/LeS8Bajjb4cBi57v4BBjh7H8DuMPZvhN4w9keAUwvLM/lkK/3gKE+0lfk3/6DwFTgW+d1pX5XwVBCyFsT2hiTCbjWb65Mg4H3ne33gSFu+z8w1iKgjog0Kqs3Ncb8gp2evDR5GQjMNsYcNsYcAWYDCeWQL3/y1uk2xmwHXOt0l+nv2Riz1xizwtk+DqzHLv9aqd9XIfnyp0K+Lyc/xhhz4v/bO5cQOaooDH8HDFGjYHwQEgQlighJNIYohkRJ4oO8cCuufK00LuImQRJ0o6AJ4giKC0VBIwOODxAEEeMsRCOD0TgZF8YY3Tgxg4tEQQjqHBf3FFNdzvSkJ9V1e5j/g6JudfXU/et0VZ++59TcE5vzYnFgI6nWOvzfZoUt3wXuMGutxV7RXLeuqWjkszSzK4GtwGuxbWS21VxwCJPVhG53A9WNA5+Y2SFL9aEBFrn7iWj/BiyKdg6tnWppUuNjMWR/vQjN5NAVw/ObSL8se8ZeFV3QA/aKEMhhYIz0hfkTcMrd/5mkn5Za7EC5Fnut2qq63L2w2TNhsxfMbH5VV6X/unX1ATuB8di+jMy2mgsOITfr3H0VsBnYbma3l3d6Gvf1xLO/vaQFeAW4BlgJnACezyHCzC4C3gN2uPsf5X057TWJrp6wl7v/6+4rSWVxbwGuz6GjSlWXmS0HniDpu5kUBtrVlB4z2waMufuhpvo8G+aCQ8hav9ndf431GPAB6SY5WYSCYj2WUWunWhrR6O4n4yYeB15lYhjcmC4zm0f60n3b3d+Pl7PbazJdvWCvMu5+ChgE1pBCLkV1xnI/ndZir1PXpgi/ubufAd6gWZutBe4xs19I4bqNwIvkttVMkw+zZSGVCT1OSrgUybNlDfW9ALi41P6SFHPcR2ticm+0t9KazBrqgqaraU3edqSF9EvqZ1JSbWG0L+2CrsWl9uOkOCnAMlqTaMdJCdJaP+c47zeBvsrrWe3VRldWe0VfVwCXRPsC4HNgGzBAa6L00WhvpzVR+k47zV3Qtbhk0z7g2UzX/nomksp5bXWuJzMbFtJTA0dJ8czdDfa7ND6s74Dvi75Jsb8DwI/Ap8VFFRfgy6HzCLC6Zj39pHDC36RY48Mz0QI8REpeHQMe7JKut6LfYeBDWr/wdoeuH4DN3ficgXWkcNAwcDiWLbnt1UZXVnvF8W4Avg0NI8CTpftgKM5/AJgfr58f28di/9LpNNes67Ow2Qiwn4knkRq79uOY65lwCFltpakrhBBCAHMjhyCEEOIskEMQQggByCEIIYQI5BCEEEIAcghCCCECOQQhOsTMdpjZhbl1CFE3euxUiA6J/y5d7e6/59YiRJ1ohCBEG8xsgZl9FHPpj5jZU8ASYNDMBuM9d5vZQTP7xswGYp6hohbGXkv1MIbM7Nqc5yLEdMghCNGeTcCou9/o7stJUxyMAhvcfYOZXQ7sAe70NInh16Q57gtOu/sK4KX4WyF6FjkEIdpzBLjLzJ4zs9vc/XRl/62kIiVfxPTK9wNXlfb3l9Zruq5WiHPgvOnfIsTcxd2PRgnFLcDTZnag8hYjza9/31SHmKItRM+hEYIQbTCzJcBf7r6fNNPpKuBPUvlKgK+AtUV+IHIO15UOcW9pfbAZ1ULMDI0QhGjPCmCfmY2TZmN9hBT6+djMRiOP8ADQX6q4tYc0iyjAQjMbBs4AU40ihOgJ9NipEF1Cj6eK2YZCRkIIIQCNEIQQQgQaIQghhADkEIQQQgRyCEIIIQA5BCGEEIEcghBCCAD+A3rralb7agnlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEWCAYAAACdaNcBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd4m9XZ+PHvkSzLe4/EdmI7iZ29BykJEEKAsFJ4KaODFihQoLR0/jro21JaWmjf7re8dDBbNrTMAGUlrJC9EzvLI957akvn98eRZDtxbCdxYse5P9ely9KjR4+OHOf27fsspbVGCCHEyGIZ6gYIIYQYfBLchRBiBJLgLoQQI5AEdyGEGIEkuAshxAgkwV0IIUYgCe5CCDECSXAXYpAppfKUUqVD3Q5xepPgLk5bSqmIoW5DX5RS1qFugzh1SXAXw45SqlQp9QOl1C6lVLNS6hGlVFS35y9VSm1RSrUopT5WSs3o9tz3lVL7lVLtwddf0e2565VSHymlfqeUagTuVkpNUEqtVkq1KqUalFLPdDv/TKXU+uBz65VSZ3Z7bpVS6mfB67Urpf6jlEo7wuf5nlKqMnhesVLqvCOc96hS6v+UUiuVUp3Aucf3nRSnNa213OQ2rG5AKbADGAOkAB8BPw8+NxuoA84ArMCXgufbg89fBWRhEpdrgE5gdPC56wEf8DUgAogGngLuCp4fBSwOnpsCNAPXBc/9bPBxavD5VcB+oDB4nVXAfb18lonAQSAr+DgPGH+Ez/0o0AosCrVnqP8t5Hbq3iRzF8PV/2qtD2qtm4B7McEV4BbgL1rrtVprv9b6McANLATQWj+nta7SWge01s8Ae4EF3a5bpbX+k9bap7V2Al4gFxN8XVrrD4PnXQLs1Vr/I3juU0ARcFm3az2itd4TvM6zwKxePocfsANTlFI2rXWp1np/H5/7Ja31R8H2uwb83RLiEBLcxXB1sNv9Mkw2DiYQfztYkmlRSrVgMvwsAKXUF7uVbFqAaUDaEa4L8P8ABaxTSu1USt0YPJ4VfN/uyoDsbo9rut13AHGHfgit9T7gG8DdQJ1S6mmlVNah5/XRPiGOiQR3MVyN6XZ/LFAVvH8QuFdrndTtFqO1fkoplQv8DbgDUz5JwpR3VLdr9VgGVWtdo7W+WWudBXwFeEApNSH4frmHtGksUHm0H0Rr/aTWenHwehq4v6/Tj/b6QvRGgrsYrr6qlMpRSqVgauKhjs6/Abcqpc5QRqxS6hKlVDwQiwmO9QBKqRswmfsRKaWuUkrlBB82B18fAFYChUqpzymlIpRS1wBTgFeP5kMopSYqpZYqpeyAC3AGry/ECSXBXQxXTwL/AQ5gOi5/DqC13gDcDPwvJhjvw3SUorXeBfwGWAPUAtMxnbF9mQ+sVUp1AC8Dd2qtD2itG4FLgW8DjZjyzaVa64aj/Bx24D6gAVPGyQB+cJTXEOKoKa3lr0AxvAQnAN2ktX57qNsixKlKMnchhBiBJLgLIcQIJGUZIYQYgSRzF0KIEWjIFk5KS0vTeXl5Q/X2QghxStq4cWOD1jq9v/OGLLjn5eWxYcOGoXp7IYQ4JSmlDp053SspywghxAgkwV0IIUagfoO7UuphpVSdUmrHEZ5XSqk/KqX2KaW2KaXmDH4zhRBCHI2B1NwfxUz1fvwIz18EFARvZwD/F/x61LxeLxUVFbhcI3ul06ioKHJycrDZbEPdFCHECNVvcNdav6+UyuvjlE8Dj2szYP4TpVSSUmq01rr6aBtTUVFBfHw8eXl5KKX6f8EpSGtNY2MjFRUV5OfnD3VzhBAj1GDU3LPpuQZ1BT3XvA5TSt2ilNqglNpQX19/2PMul4vU1NQRG9gBlFKkpqaO+L9OhBBD66R2qGqt/6q1nqe1npee3vswzZEc2ENOh88ohBhagxHcK+m5sUIOx7ChgRBCjERef4DHPi5l7YFGAoGTt9zLYAT3l4EvBkfNLARaj6XePhy0tLTwwAMPHPXrLr74YlpaWk5Ai4QQx6LV4eWFjRX4T0AwrWl18Ye399Lh9tHq9PLSlkrcPn+v52qt+e8Xd/CTl3dyzV8/YdH973Lva7vYV9c+6O06VL8dqkqpp4AlQJpSqgL4CWAD0Fo/iNmx5mLMpgkO4IYT1dgTLRTcb7/99h7HfT4fERFH/latXLnyRDdNCDFAgYDmjqc28cHeBmLtVpZPG31M13l9ezVPrz/Iw9fPx2rpKqX+z3+KeX5jBW/srKHT7aO8ycGMnER+f80sxqX33Eb30Y9LeXr9Qb5y9jimZCXwytYqHv24lIKMeCZkxB/X5+zPQEbLfLaf5zXw1UFr0RD6/ve/z/79+5k1axY2m42oqCiSk5MpKipiz549XH755Rw8eBCXy8Wdd97JLbfcAnQtpdDR0cFFF13E4sWL+fjjj8nOzuall14iOjp6iD+ZECNTq9PLtooWziro6sP783v7+GBvA5FWC89vrGTppEzeK67j/MmZWCwD7+968P0DbD3YwtaKFmZkJ1Le5CAuKoKXt1SxcFwK2ypaSYiy8cOLJ/Gnd/ex7LeruXDqKG46axxzc5Mpb3Rw3+tFnDcpg+8tn4TFovj0rGxaHB4iI058d+eQrS3Tn5++spNdVW2Des0pWQn85LKpR3z+vvvuY8eOHWzZsoVVq1ZxySWXsGPHjvCQxYcffpiUlBScTifz58/nyiuvJDU1tcc19u7dy1NPPcXf/vY3rr76al544QW+8IUvDOrnEOJ0oLXG4w9gj7D2OF7e6GBTeTOXz87msY9L+e1be3jipjNYNCGNunYXf3pvH5fMGE12UjQPf1jCd57bystbq/jbF+dx/pTMAb13SUMnWw+aUut7RXWsKqrjj+/uY2JmPB5/gF/+1wyibVZi7FYSomxcPiubRz4u5YlPynh9Rw1zxiahgQiL4udXTOvxSyUpJnLQvkd9keUH+rBgwYIeY9H/+Mc/MnPmTBYuXMjBgwfZu3fvYa/Jz89n1qxZAMydO5fS0tKT1VwhRpSfv7abJb9exYH6Dv76/n6++cwWHB4ftz+5kW88s4WmTk84AP/0lZ34/AEe+rAEnz/Ady+YyJVzcvAFNC9vrQLg3aLaAb2vP6B5aUslSsH49Fhe31HDPz4pY1RCFMW17SybnEF+WiyjEqNIiDITETMSovje8kms+cF53H3ZFOo73Gwub+HbF0xkdOLQ/OU+bDP3vjLskyU2NjZ8f9WqVbz99tusWbOGmJgYlixZ0utYdbvdHr5vtVpxOp0npa1CDIXXt1fztw8OUJARzx1LJzAmJWbQrr22pJHqVhfL//ABHl8AgE3lzZQ1OgDYWNbMtspWcpKj2VPbwdef3szq4nounZFFXpr5v7togvnLOiYygneL6qhudfK9F7YzIT2OK2ZnMz0nMfx+Na0uvvnMFjaWN2OzKBbmp7JkYjq/fL0IgGdumUusPYLspCMH61h7BNcvyue6T+VRVNPGlNEJg/b9OFrDNrgPhfj4eNrbe+/Fbm1tJTk5mZiYGIqKivjkk09OcuuEGH5e217Nzqo2NpW3MDopim8sKzyu660raaLN6WXJxHT21HZw3qQMKlucXDHbzIv85etFnDk+lfWlTby2rYr6djc/uWwKFc1Onl5XjssX4LYl48PXe+yGBVgtiuc3VvDWrlpufnwDxTXtrD3QyD8+KeWP187moumjWV/axFf+sRGnx8/ls7LYVtHKlxfnk5cWwy9fL2LK6AQW5KcMeI6K1aKYmpXY/4knkAT3blJTU1m0aBHTpk0jOjqazMyu+tzy5ct58MEHmTx5MhMnTmThwoVD2FIhTg6tNR/vb+RT41J77Ywsb3KwID+FHZWtNHS4j+u9fP4A33xmC06vnydvPgOPL8BlM7O4PBjYtdaMT49jbm4yNz62nte2mxHXM3KSuGFRPt+9cCINHW5ykrv+eoiwmsrzkokZAOyobOObywq5/sw8bnxsPV99chPTc5LYXdVGTnI0z37lU0zI6BrxorXmhkV5nD8585SbfCjB/RBPPvlkr8ftdjuvv/56r8+F6uppaWns2NG1eOZ3vvOdQW+fECfTpvJmPv/3tfzumplcMTvnsOdLGzpZMSuL2jYX9e1HDu5aa6756yfMzU3me8sncfsTG0mNtfOzy6eFz3l7dx2VLaaM+cLGCgAmdytrKKVYFuwQnZebzObyFqwWFS59RNmsPQJ7d+nxduaMTaK+w81XzhlHlM3KP768gD+/t4/1pc2cPyWTe6+Ydlhnp1JqWJSIj4UEdyHEEe2t7QBg5faaw4J7i8NDm8tHbkosJQ2dfQb3HZVtrCtpoqrFyVfPncB/dtZitSi+d9Ek4uwmDD36cQlpcXYaOtw8te4gkREWxqXH9nq9ubkp/O2DEgoy4oiOtPZ6zqEevG4uaPNLAEwd/rsXThrQa09FMlpGCHFEJQ2dALy/p55Ot6/Hc6GOzbGpMaTH2anvoyzzwiaTiVc0O3lhYwW+gMbtC/CfnTUA7K/v4JMDTdx8Vj4TM+PpcPsozIzDZu09RM3NTQZgRs7A69oZ8VFkJEQN+PxTnQR3IcQRHWjoxGZVuH0BVhX3XMm1rMkE97zUWJNxt3swcxp78vgCvLy1Klw++dO7+7BHWMhKjOKlLWaY4rYKM6Rx6aQMFhekAfQ50iQ93s7dl03hxsWybPaRSHAXQoQdGpxLGjo5pzCd1NhIXt/Rc8mo8kaT1Y9NiSE93o7T66fTc/gaK89sOEhTp4dvX1BIZoIpu8zLS+bTs7P5cF8DjR1u9tR2YLMq8tJiWTzBBPfJ/QwjvH5RPpNGDd1Qw+FOgrsQgt3VbVz9lzVMv/s/fOPpzYCZzFPe6GB8RhznFKbzyYGmHsG/tNFBRryd6Egr6fFmfkeo7u70+Hl9ezW//U8xP35pB2eOT+WcwnQWjTeBe9GENJZPHYU/YEbj7KlpZ3y6KcMsmpDG15ZO4LKZWSf5uzCySHAXYoTy+gN8+n8/7HdmZm2bixseWc+B+k5mjUnixS1VfLC3nqoWJx5/gHFpscwck0RDh5uatq6Je+WNDnJTzeiU7sHd5fVz/SPruO2JTfzx3X0snZjBw9fPJ8Jq4ZyJZg2YswvSmZqVQEyklQ2lTRTXtlOYaRbSioyw8O0LJpIWZ0ccOxkt04+7776buLg4GdYoTjkVzU62VrSytqSJpZOOvKbKt57dQpvLy3O3mjHeF/zufe5+eSc/uGgyAPlpcURYzRjvbRWtlDU62FjWzIFgyQYIB+KGDjfffX4ba0uauO+/prNkYgaZCfbwGPHLZmRRkBHPlCxTTpkzNpnVe+qpaHby2QVjT9j34nQkwV2IEao82OFZ13bkUSxef4B1JU3cuCg/PKPyJ5dN4cZHN/CLlbsByEuLISHKhtWi2F7Ryqo9deyoNIv6HZq5765u45WtVdy2ZDzX9hKsLRYVDuxgRr18uK8BIJy5i8EhZZle3HvvvRQWFrJ48WKKi4sB2L9/P8uXL2fu3LmcddZZFBUV0draSm5uLoGAWfeis7OTMWPG4PV6h7L5QgDdgnv7kffrLWvsxOvXTBzVFViXTsrk4umjONDQSZw9gvQ4O1E2K4WZ8azcUc2OyjaumpvDhVMzuXDqKACSYyKxWhSv7zBDGwe6+uL8vJTw/YkS3AfV8M3cX/8+1Gwf3GuOmg4X3dfnKRs3buTpp59my5Yt+Hw+5syZw9y5c7nlllt48MEHKSgoYO3atdx+++28++67zJo1i9WrV3Puuefy6quvcuGFF2Kz2Qa33UIcg9Bolto+MvfQJKVDs+a7V0zlw70N5KXFhksqM7ITeWbDQQC+cX5hjwW0rBZFamwk++o6sEdYmDbAdVVmjU3CalFEWi3kJMu+B4Np+Ab3IfLBBx9wxRVXEBNj/txcsWIFLpeLjz/+mKuuuip8nttt/sNcc801PPPMM5x77rk8/fTTh+3iJMRQCWXutW1Hztz31HYEl7btuYNQRnwU/7zpDBRd66lMzzHBfc7YpF5XRkyLs1PX7mbWmKQBb0YRZ49gWlYCFos6qo00RP+Gb3DvJ8M+mQKBAElJSWzZsuWw51asWMEPf/hDmpqa2LhxI0uXLh2CFgpxuPIms05Lu8uH0+Pn9ic2ctW8MVw8vWvbuT117YxJjul1Cv+MnKQej2eNMY8vndH7EMX0eDtUw7y85KNq5x+unc3J2zb69CE190OcffbZvPjiizidTtrb23nllVeIiYkhPz+f5557DjATPbZu3QpAXFwc8+fP58477+TSSy/Fah3YOhdCnEhaaw42OUiIMvnb9spW3iuu583gdP+QvbXtFGbG9XaJw0zLTuSxGxfwhYW5vT4f6lSd162OPhB5abHkp/W+how4dhLcDzFnzhyuueYaZs6cyUUXXcT8+fMBeOKJJ3jooYeYOXMmU6dO5aWXXgq/5pprruGf//wn11xzzVA1W4iwd3bXcrDJSYfbFw60HwVHpBTXdO1X4PUHKGnopOAoOjLPKUw/YslldGIUVotiztijy9zFiTF8yzJD6K677uKuu+467Pgbb7zR6/mf+cxnel1TQ4iTraLZwZcf28C0bDPccG5uMu8W1YWD+4H6Trz+ADarJTxSZqCZe39uWJTPoglpJEbLgILhQDJ3IUaQ0EqNoXHooaGGm4N7jXr8AcqCo2j2BEfKFGQMzhDElNhIFo5L7f9EcVJIcBdiBAmNkIkIjjyZnp1IpNWCP6DDGXVxjQnqH+9vICbSSsEgZe5ieBl2wf10KG+cDp9RHL+aVhetzqObEFfe5MBmVXz/okksmZjeY1Gv8yZnYFFQXNuO1pp3d9exeEIa9ggZBDASDavgHhUVRWNj44gOflprGhsbiYo6fTYNEMfmhkfXc/8bRUf1mvImBznJMdx01jgevWEBAJkJJrhPz04kLzWWPTXtFNe2U9Xq4rzJGYPebjE8DKsO1ZycHCoqKqivr+//5FNYVFQUOTmH70cpRHf17S6qg3uKHklDhxu3LxCeVHSwycGYlJ77iGYGdx8qzIynMDOePbXtvLO7DoBzJ0pwH6mGVXC32Wzk58vOKkIAdLr9/ZZl7n55J6WNnbz6tbMA06F66NZzGcGyTEFGHBNHxfPGzhoeXL2f6dmJp9W2c6ebYRXchRCGP6BxevsP7pUtTvbVdaC1ps3po9XpZewhmfuyKZm0Or2kx9v5/BljaXf52FjWxHWfyjuBn0AMNQnuQgxDTq/Zrq7N5evzvKZODy5vgPp2N3XBXZAODe5nFaRzVoFZdz0jIYofXzblBLRYDDcD6lBVSi1XShUrpfYppb7fy/NjlVLvKaU2K6W2KaUuHvymCnH6cLhNUO8vc2/q9ACmIzU0DPLQmrs4PfUb3JVSVuDPwEXAFOCzSqlDf/X/CHhWaz0buBZ4YLAbKsTpJLTRtMcXwOXtuen07uo2dle34fEFaA9m9t2D+6GZuzg9DSRzXwDs01of0Fp7gKeBTx9yjgZC26skAlWD10QhTj+d7q5yTNsh2fvXntrMj17cQbPDEz5W3uSgrLGT5Bgb8VEy/V8MLLhnAwe7Pa4IHuvubuALSqkKYCXwtd4upJS6RSm1QSm1YaQPdxTieDg8Xdl699JMaUMn++o6qGpxhksyYIL7xrJmph+yTK84fQ3WJKbPAo9qrXOAi4F/KKUOu7bW+q9a63la63np6emD9NZCjDydnq7MvXtwf6fIjE+va3dTH+xAjbRa2FLewp7aDhaOO7rldsXINZDgXgmM6fY4J3isuy8DzwJordcAUUDaYDRQiNORw9175v7O7lrADJUMLd87JSuBAw1mMTBZuEuEDCS4rwcKlFL5SqlITIfpy4ecUw6cB6CUmowJ7lJ3EeIYdc/c21ze8Nd1JU3hjaR3VZuVH0M7JMVGWpmePbC9S8XI129w11r7gDuAN4HdmFExO5VS9yilVgRP+zZws1JqK/AUcL0eyQvECHGCObp1qLY6vHj9Ab73/DZ8Ac2XzswDYGdVK0oRnpE6Ly8Fm3VYLRclhtCAJjFprVdiOkq7H/txt/u7gEWD2zQhTl+dPTpUffz4pZ28vqOGH10ymeXTRvHDf29nf30nSdE28oJb1ElJRnQnM1SFGIYcHh8RFkVkhIVWp5e3d9dy2cwsbjprHFprIiMseHwBUmIjmZGdyFfPHc9V82QxOtFF/oYTYhjqdPuJibSSGG2jssVBfbubKaPNVBKlFKMTzYJfqbF2IqwWvnvhJNLi7EPZZDHMSHAXYhhyeHzE2iNIjLaxudxskTc+PTb8/Kjgao7JsTJhSfROgrsQw1Cnx090pJWEaFt4QbBx6V3b4YUy95RYydZF7yS4CzEMOdw+YiMjwvueWi2qx5oxmeGyTOSQtE8MfxLchRiGOj2m5p4QXCcmNyWGyIiu/66jE0KZuwR30TsJ7kIMA/6Apt3VNRO1e80dYFy3ejvAqESzrZ4Ed3EkEtyFGGLbKlr49J8/5Mz73qUjOHnJ0W20DMD4bvV2gMmj44m0WijIjDvsekKAjHMXYkh5/QE+//e1eHwB3L4ARdVtzMtLodMTqrmb/6KHZu65qbHsuudCImRGqjgC+ckQYgj4A2Z1jrLGTtpdPr5+XgHQtV6Mw+Mnxm4lMSZUljk8Q5fALvoiPx1CnARaa/5v1X6qW500d3qYf+/bPLfhIMU1HQCcU5hOYrSN3dVtaK1xePzERkawdFIm37mgkNljZJ12cXSkLCPESbC3roP73yhie2ULU7MSaer0sKq4nvEZcVgUTMiIY/LoeHZVt+P2BfAHtMnco23csbRgqJsvTkES3IU4CUJrr6/cXsNH+xoB2FDWhC8QIC81liiblcmjE3hqXXl4X9QYm3XI2itOfVKWEeIk2FPbjtWiiI+KoNXp5dyJ6dS2uVmzv5HC4Prsk0cn4PIGwnX3GLvkXuLYSXAX4iQormknPy2W/7d8EksnZfCt8ycC0ObyUTjKBPfQwmAbSpsAiI2U4C6Onfz0CHEC/PzVXVS1Onng83MBk7lPzUrkuoW5XLcwF58/QEykFYfHH95ZqSAzjgiLYu0BE9xj7FKWEcdOMnchToB3i+tYs9/U1p0eP2VNjnD5Bcwwxpk5ZgTMxFFmmKM9wsqiCWmsk8xdDAIJ7kIMMofHR0lDJ80OL20uL/vqOtC6K4iHnF2YTkpsJLmpXROUbj5rXPh+TKRk7uLYSXAXYpAV17QT2kH4YJOD4lozUqZ75g5w81n5rPrukh77ni6akMqkYA0+VjpUxXGQ4C7EINtd3R6+f7DJwZ7adiIjLD0ydDClmdCqjyFKKb55fiGjEqLIiJe12sWxk9RAiEG2u7oNe4QFty9AeZODHZWtTMyMx2pRA3r9hVNHceHUUSe4lWKkk8xdiEG2q7qNmTlJJEbbKG10sK2ilVmyfIA4ySS4CzGIAgFNUXUbk0fHMzYlhtXF9XS4fcyU4C5OMgnuQgyig80OOj1+Jo9OYGxKDJUtTgBmjUkc4paJ040EdyEG0e7g0gGTRycwJrjnabw9gnFpsqmGOLkkuAtxHNpdXhweX/jxrqo2LAomjooPb2g9Y0wilgF2pgoxWCS4C3GMtNZc85dPuO2fm8LHdlW3My49jiibNRzcQzNRhTiZZCikEMdoR2WbWcGxGvbVdTAhI47d1W3MyU0GzD6noxOjOG9y5hC3VJyOBpS5K6WWK6WKlVL7lFLfP8I5VyuldimldiqlnhzcZgox/LywqYJIq4VIq4V/flJGq9NLZYuTyaPNDNPUODtrfnAec4PBXoiTqd/MXSllBf4MnA9UAOuVUi9rrXd1O6cA+AGwSGvdrJTKOFENFmI48PoDvLK1imVTMoi0WnhhYwVnFaQBXUv3CjGUBpK5LwD2aa0PaK09wNPApw8552bgz1rrZgCtdd3gNlOI4eXj/Y00dnq4YnYONy7Op8Pj43svbAckuIvhYSDBPRs42O1xRfBYd4VAoVLqI6XUJ0qp5b1dSCl1i1Jqg1JqQ319/bG1WIhhYHN5M0qZhb5m5CTx1SUTaOhwkxobSbqsCSOGgcHqUI0ACoAlQA7wvlJquta6pftJWuu/An8FmDdvnh6k9xbipNtR2cb49Dhigmuuf2NZAZsPNpMWZ0cpGfYoht5AgnslMKbb45zgse4qgLVaay9QopTagwn26wellUIMMzurWlmQnxJ+HGG18M8vnzGELRKip4GUZdYDBUqpfKVUJHAt8PIh57yIydpRSqVhyjQHBrGdQgwbDR1uqltdTMvquaSAUkqydjFs9BvctdY+4A7gTWA38KzWeqdS6h6l1IrgaW8CjUqpXcB7wHe11o0nqtFCDKWdVWaJganZ0nEqhq8B1dy11iuBlYcc+3G3+xr4VvAmxIi2o7IVgKlZshiYGL5k+QEhjtKuqjbGpsSQGG3r/2QhhogEdyGOgtaazeXNTJOSjBjmJLgLcRS2VrRS1epi6SRZL0YMbxLchTgKr22rwmZVnD9FgrsY3iS4CzFAWmte21bN2QXpUm8Xw54EdyEGaPPBFqpaXVwyY/RQN0WIfklwF2KAdgaHQC6akDbELRGifxLchRigNpfZTk9KMuJUIMFdiAHqcPuwWRX2CPlvI4Y/+SkVYoA6XD7i7BGyfow4JUhwF2KAOtw+4qOkJCNODRLchRigdpeXOLvsKS9ODRLchRigdpePuCgJ7uLUIMFdiAHqcPuIl8xdnCIkuAsxQKbmLsFdDIDW4HMPaRMkuAsxQFKWEYfZ/Sr8ZjIUv2ECevkn8PTn4f5c+EU2rH8InC3wwW9hwyPQuP+kNU1+UoUYIDMUUkbLjBgd9fDM52H5fZA9p+u4pxNqdsDoGWCL7jruc0PRq5BaYJ5rrYCXbgd3Ozz9OUjJh8Z9EJ0CU//L3H/t2/D+r6G92lzDYoMLfgZn3AoneEitBHchBsDt8+PxB6QsM5JseAgOroXV98Pnngkeexhe+w5oP0w4Hy7+NbxwE7haTRDvqAFlMcG7cgME/HDLalh1H7ha4FN3wIyrITIWvE548mpoLoMvvwUxqfCfH8Eb3we/BxbdeUI/nvykCjEAHcGlByS4n2LcHWCJAFtUz+M+N6z/O1jtsOcNqC+GpFx4917Img3jlsDjMq2iAAAgAElEQVQH/wMPLIQIO+SfYwL53C/B7ldgxwvmvEt+Y7L4zz55+HvbouG6l8x9S7ACfu2TsOkxmHL5ifzUgAR3IQakPRjcZZz7MOT3wdoHIWc+jD3DHKveCh/90ZRRkvPhiy9BfKY5t7kUdjwPnfVw5UPw0ldNTTz3THA0wGcehnHngN8N254zWX3WrK73K7wQVvxpYGUVyyHdmkrB3OsH65P3SX5ShRiADrcE92FHa2jYA2/eBfveguhkuPFNE6i3PQ32RJj+Gdjxb/jrErDHQVMJBLzm9dlzYdqVULkJPvmzCfijpkP+2eb5C34Oy+45PEDDCa+XDwb5SRViANrDZRnpUB0Ufi8oa++BM6RhH7jbenZ2ggnqu18xQb213JRdlvwAPv6TKaMAnPVtOPPrEJ0Es78I79xj7k+6xHSIphWYQK6UCeJpBab2vuQHPQN3X+0b5iS4CzEAocxdau7HoHyt6UQccwaccQsk5cEjF5tg+7lnD8+CtYZtz8Ird5p69/87AG2VJkufsAw++gO8/RPInAbn/MnUw5NzIWMKvPcLuOh+U1YJGXsG3PDakdtnscC8G8xtBJGfVCEGoN1l/pSXsswx+PC3ULcbanfC3v+Y7LhinXluxwumdBJSvQ3e+AGUfQjxo80Qwtod8OHvYOeL8Nmn4P3/gYILTeektdu/x5QV5iYACe5CDEi45i6Z+9FpOWgC+uJvQt5Z8I/L4cXbIGW8qYG/eZfJyMcsgJRx8PgKMxb8ol/DxOXw++mw/13Y9w6gzXhyreH8n/YM7OIw8t0RYgDaZSjkwOhgAB49E875Hmx63Byb8yVTOpn1edjyBJz9HciYbGZzrr7fvDYyDqw2uPENSB1vjqWMhzUPmNr7/Jth/d9g5mfNa0Wf5CdViAHocPuItFqwR1iHuinDg9ZQ9hFkzYHImK7jdbuheKW5Fb0K9XtgwnkmsANc9CsoOB8mrwCLFb61C3weM6Fo42Nm0lAosAPkLTbjwq12k63PvNbU1kW/JLgLMQDtLu/pUZIJBHofIdJRD+v+ArteNkMI26vgwCoYv9R0ilqDo4hK3jdfP3WHKaXM/RIs+kbXdexxMPWKnteOiISFt5nbofLOMsE9/2wz6zNn3qB8zNPBafDTKsTxC22xNyJ9+DtT7568Ah67DFLHmUk6IY4mePgCM0Y890wzDBENs6+Dzf+AV78Bn/6zObdktZk0dOG95na88s+CiCiYeuJndI40A/ppVUotB/4AWIG/a63vO8J5VwLPA/O11hsGrZVCDLERu9xvwz54+26ISoKAz4xSqdsJl/7BZPB+LzxznVkk64aVJrh7naADJpOOSTFDE8+4FdInQ+mHh2fmxyN+FHxzl3kfcVT6HaGvlLICfwYuAqYAn1VKHVb0UkrFA3cCawe7kUIMtfaRmrmv/T8zOsXVAv/6ijnmbIaabeb++r+bgL/iTyawg1kzJTLW3F/8TbDFmqn+NVtNx2dohudgiU09JWaEDjcDmX61ANintT6gtfYATwOf7uW8nwH3A65BbJ8Qw0K7a4Rk7jtegIeXm05MRxNseRJmXgPTrzbT8hd/05xX8r55ftV9MO5cmHFN79eLTjZ19R0vwOvfM8cGO7iLYzKQn9Zs4GC3xxXAGd1PUErNAcZorV9TSn33SBdSSt0C3AIwduzYo2+tEEPElGVO8aUHmg7Ay18HTwdUbTIThrwOWHi7WY42MQfO+X9QtNLUzptLzFK3F/y878x54e1mU4rmMrNKYlzGyftM4oiOOxVRSlmA3wLX93eu1vqvwF8B5s2bp4/3vYU4Wdpc3lO7LBMImLKLCv6xXvohVKw3HamZU82xZT8xX/PPNuPJARZ+FUZN6/vaSWPgzi1mk4pDl9YVQ2YgZZlKYEy3xznBYyHxwDRglVKqFFgIvKyUkjFLYkTodPtocXgZnXQKB66tT5op/xf9yowTP7AKSj4wQxkPVXC++Tr3BpO1D0RClgT2YWYgqch6oEAplY8J6tcCnws9qbVuBdJCj5VSq4DvyGgZMVJUtjgByEmO6efMYahqixkF8849Zr3zmddC5cauzLy34F54Idy2BtInndKrIp7u+g3uWmufUuoO4E3MUMiHtdY7lVL3ABu01i+f6EYKMZQqmh0A5CRH93PmMdJ68EeDNO6H//w3FHdbDfHap8z75C0ywV1ZzSSh3mTKLNBT3YCKiFrrlcDKQ479+AjnLjn+ZgkxfFQ0hzL3ExDcO+rh7+eZOvelvz/+xbD8PrOz0LZnwBYD5/3E1NVtMZAz15yTu8h8HbMAohKO7/3EsHUK9xAJcXJUNDuxR1hIj7MP7oW1hte+aSYIbf4HeDrNFm/Hk8VXbjS7EM29wSytG595+DlxGbDgK2bdFjFiSXAXoh8HmxzkJEejBrt0svPfZir/srvNmPKP/wgX/gISRh/7NUuDa7ss/W8z+edILv7Vsb+HOCVIb4kQ/ahodg5+Z6rPA+/8FDKnw6e+1lX7bikzX/1eeO56WPuXo7tu6Ydmh6K+Ars4LUhwF6IfFc2Owa+3b34cmkvhvB+bOntoSdzmYHB/56cms1/9KxPoexPw93zs85gt7aTcIpDgLkSfOtw+mh3ewc3c3R2w+tcw9lNdY8qTgjO2W8qg9COz2XPWbHA0wN63Dr9G6Ufwi2zzNaRqE/icEtwFIMFdiD4NaBhkcxn8/XxorzG182e/aFZbDHG1Qs12k1kDfPAb6KiBZT/t6jy1RUNcprnWvrfBEgFffBli0swEpO68Tnj5DhPI1/y56/iB1YDqGg0jTmvSoSpEHyqaehkG6XWZXYRCG1QcWGVmf+57ByLssOsls47LTcHH/7oF9rxhdhOadDEUvQYzroWxZ/R8s6Rck7l31JoJRFEJMONqWPc3qCuCjEnmvNX3m+vnnw17Xjf7lHbWw0e/N+ufy/K4AsnchehTV+berSzz8IVmAa6Q+mLztXKjuSmrydTf+gnU7jSBfca1MOc62Pu22Xzi/J8e/mbJuSZzr9kOo6abYwtuMSsvPnyBycyrt5nldWd9oWuDjBe+DP+8EmLT4MqHTsB3QZyKJHMXog9lTQ5iIq2kxUWaA55OqN5qAvDSu8xKivVF5rmqTWCNNJODRs8ya6XvfdNMIFr+S5NRL7vbXCN+1OFvlpQL258HdFdwT8mHm96GJ6+Gf1wOcaPMCo4X/Mxcb8rlZq/S3EVm/1FZkVEESXAXog/ljQ7GpsR0jXGvKwI0aL8pl5z/U2jYY56r2WHKNfNvgvPvgZZyM/3/jNu6SiX2eHPrTXKuuTZ0BffQ8Zvehle+ATueh6se67relX83a8dEDPIEK3HKk7KMEH0obewkN7VbSaZ2h/k6eiZsfBQ66qD1oHkc8ILPZTaQtljhMw/Bhb80a6QPRFJu1/3MQ5bZtcebQP7tPT33E7VYJbCLXklwF+IIAgHNwWYneamxXQfrdpkyy/k/M1vTffAbc3zm57rOyQ6u4WKLhk/dPvAOztBY98Qxvb9Gqd6XExCiF1KWEeIIatpceHwBxqbGmDp74hjTQZox2cwoTRprdiACs3RubAagu8asH62EHNMZ270kI8QxkuAuxBGUNZqRMhMj6uBvF8GYM6B+N0y82KxzPvOzZliixWZWXpxzHejAsS/8ZY2ARV83k5uEOE4S3IU4grLGTgCm7vi1qaeXfWieCNXDZ15rgnvqBBOYz+t1Feyjs+zu47+GEEjNXYgjKmtycG7EdqJL3oRz7+oqt4Q2skgZZ4YiFl4wdI0U4ggkcxfiCOrq6rjP9jdImQBnfh3SJ8IbPzAjY0KufmzoGihEHyS4C3Go2p2w/u98qXwdaboJrnjWbP485dPmJsQpQMoyQgB1bS42lzebB+sfQm94hAnevbyddVvX9nRCnEIkuAsBPLj6ANc/st48qC/GPWoOU9yP0Djz1qFtmBDHSIK7EEBjp5tWpxeHxwf1RdTY8wGYPTZpiFsmxLGR4C4E0OY0ux011VWBo4EiXxYxkVYKM4+wDowQw5wEdyGANpcPgM5Ks3bMJx3pTM9OxGoZ5E2xhThJJLgLAbS7TObuqzHL977TkMzssclD2SQhjosEdyGANqfJ3K0NRfhtcRz0JzNrjNTbxalLgrs4rbxXXMe/N1ccdrwtmLnHtO6jMTofUNKZKk5pEtzFaeWhD0r47Vtmc41XtlbxP28W4/UHcHj8ACQ7SiizjCUtLpLMhKihbKoQx0WC+2nm6XXlbK9oHepmDJn6djc1rS4CAc2/N1fyxNoy2oOdqQl0EO9rYrdvFOPT44a4pUIcnwEFd6XUcqVUsVJqn1Lq+708/y2l1C6l1Dal1DtKqdzeriOG3j2v7uLJdeVD3YwhU9fuwuvXNHS6qWh20Ozw0tTpAWCsqgNgc2cy4zMkuItTW7/BXSllBf4MXARMAT6rlJpyyGmbgXla6xnA88CvBruh4vh5fKb80OH2DXVThoTHF6DZYWrrVS0uKpudAJQ2mKV9J0c1AVDkSpXMXZzyBpK5LwD2aa0PaK09wNNAj9WTtNbvaa0dwYefADmD20wxGFqDE3U6gp2Hp5vGTnf4flF1G53BOvuBhg4AZsSatWUO6nTGp8cefgEhTiEDCe7ZwMFujyuCx47ky8DrvT2hlLpFKbVBKbWhvr5+4K0UgyIU3EM15tNNfXtXcF9X2hS+f6DeZO4FtgYadTwdxEjmLk55g9qhqpT6AjAP+HVvz2ut/6q1nqe1npeenj6Yby0GIJy5n6Zlme7BfUNpc/h+KLjnUMdBnUGUzUJ2UvRJb58Qg2kgwb0SGNPtcU7wWA9KqWXAXcAKrbX70OdPd1prtNZD2oZWp+k4HC6Ze1OnJ9yZeTKEgntMpJXyJkf4eKgsk+KpokxnMi4tDossOyBOcQMJ7uuBAqVUvlIqErgWeLn7CUqp2cBfMIG9bvCbeep7cPUBLvrDB0PahuGWud/+xEbufHrzSXu/UHCflpUIQLTNSrTNSkOHB5vyE+2oolxnyEgZMSL0G9y11j7gDuBNYDfwrNZ6p1LqHqXUiuBpvwbigOeUUluUUi8f4XKnraKaNopr2/EHhi57b3V0Bfeh/iui0+1jQ2kzu6vbTtp71ne4SYy2kZsaA0BOcjRp8ZEAFNhbUdpPhc6gUIK7GAEGtM2e1nolsPKQYz/udn/ZILdrxGnq9KA1NDs8pMXZh6QNrcH1U/wBjdPrJyZy6HZZ3FDWjC+gaejw0O7yEh9lO+HvWd/uJj3ezuhgPT07OZoWh5eDTU4K7Q3ghusuPoexc/NOeFuEONFkhupJ0uwwteXGjpNXYz5UqCwD0DHEdfdPDjSG75c2OPo4c/DUt7tJj7OTlWiWFchOig7/oh1vbQBg2tSZJJyEXzRCnGgS3E+Spo5QcB+6vuYWZ9cvlrYhDu5r9jeSEmtKIiWNnSflPesOydxzkmNID5ZlclUNWGyQkHVS2iLEiSbB/SRpCmbuDSdxdMih2rpn7oPcqery+rnjyU386MXt/a5d0+H2sb2ylf+abaZLlNSf+OCutQ6XZcanx2K1KCaNig9n7jO9WyF7DlisJ7wtQpwMEtxPAqfHj8sbAAaeubc4PPzpnb34/IHwsfWlTazvNvnmaLU6vURazT/5YJZltNb894s7eHVbNc9vrOCKBz7q85fH1oMt+AOaswvTyUqMovQkZO6dHj9Or5+MeDs5yTF8+L1zWTIxnbQ4O6NoJNezFyZedMLbIcTJIsH9JAhl7TDwmvvzGyv4zVt72NotC/7Zq7v4xcrdx9yOVqeX0Umm3tzhHrwlCF7bXs1zGyv42tIJ/PozM/EFNBXNR66jh9Z0yUuNJS8tlpKGExvc//FJGbf9cyMA6fEmUx+dGI1SirQ4O+dZg8MxCyW4i5FDgvtJ0NytFNN9fZO+bC5vAaC8qSvwlTZ0UtdmXt/q9NJwlPX7VqeXnGRTbx7Mmvuq4nrS4iL5xrJCsoPXDwXw3tS0uQDISLCTnxbbb+autebhD0s42HRsHa9PrS1nzf5GlOKwDa/T4iI5z7KJlqgcSJ94TNcXYjiS4H4SNHYL7g0DyNy11mwoM+WX0EiSVoeXNpeP+nY3Wmt+9OIObn58w1G1o8XhJSfJjPEezLLMvroOCjLisVoUOcHOysqWvoN7coyNKJuV/LRYWhxemjs9dLp93Pz4BjaWNfc4f2NZM/e8uovH15QeU/vq2l1cNS+H4p9dxLTsxB7Ppdn9LLLspCL9HFAyK1WMHBLcT6B9dR18vL8hnLlnJUYNqOZe1eqiNpihh6bJh756/AFanV7213VQXNM+4MlILq8fty9AVjD4drh9VDQ7whObjpXWmv11HUwITvxJi7MTabX0mbnXtrrCuxzlpZrVF0saO9lQ1sxbu2q54ZF1PSY3PbvBrFu39Rg2GfH6AzR0eMiIjyIy4vAf97Eta7ErL/EzLzvqawsxnElwP4F+//Ye7nhyczhzL8iM75HFH0koc02JjQyXLMq6lWfq2t1UtzpxePwDuh50jZRJiYsk2malw+3juofW8e3nth7VZzpUXbubdrcvHNwtFkVWUlS/mfuo4FjziaNMmWRnVRs7q0zwjo608vWnTB280+3j1W3VKAU7KluPeoZvXXDJgdD7Hcq27w2wJ5I7W+bhiZFFgvsJVNPqoqnTw56adiwKxqXHDqhDdVNZM9E2K+dNyqCssWfmDlDe6AhvOlHeRx261eHlL6v384W/r+Xfm81ab4nRNuKiIqhtc1HS0Ml7xXXUBWvg/XljRw1/fGdvj2N7a82iWwXdpuxnJ0f3Gdxr21yMCmbuOcnRpMVFsqW8hV1VbeQkR3PT4nHsreugrt3Fa9urcXj8XDt/LA6Pn/31HQNqa/f3AshM6GVWcCAAe96EgmVglYlLYmSR4H4ChbLGNQcaSY6JJD3eTofbh8vrD5/T6faxsazn8MZ1JU3MHJPI+Iw4mjo9tLm8lDd2BfFtlV3lie7HD/WX9/fzy9eLWFfaxB+CQTkp2kZ8VER4LLo/uJdof7TW/OrNIn771p4eNfF9de0A4cwdzMzPI5VlPD5TJgmVZZRSzBqTzOaDzeyqamPK6ATm5CYDsKmshZXbq8lNjeHGRXmAGUbZ9d4dfLi3ocfM20PVhYN7L5l75UborJdRMmJEkuB+gmitw1ljeZOD5NhI0mJN9ti9lPL4mjKu/ssntAV3R/p4fwO7qttYPnUUecEFrsobHZQ3OcLZ8baKrgDXPXNfub26x4iSvXUdFGTE8b3lk3AEdx1KjLYRb4/gQHD4YVqcnec2VvRbu99V3RZe9/z+N4rC5++r7yA+KiI8xBAgOymGunY3bl/XL7HmTg+/f3sPVcGMvnuZZPbYJA7Ud1LS2MnUrESmZScQabXw0b4G1uxvZOmkDMalxxEbaWVb8JfS3z84wLLfruYLD63lvteLjtjumtY+gnvxSlBWk7kLMcJIcB8EvQXGNpcPt69rAlJKbCSpcWaqe/dO1X11HfgDmro2Mwrmd2/tITPBzrULxpIb7GwsbeykvMnBlKwEom3WcPZqtahwcC9t6OT2JzZx4e/f57lgB2RJQyf5abFcMTsbm9WMBAmVZcAMDrnzvAnsq+vgqgfXsLe2/Yif8eWtVURYFN9YVsC6kiY+3t8Ybn9BRhyq20iT0HDI6paucs8bO2v4/dt7+demCoBwWQZg9pik4PcRpmQlYI+wMj0nkWc3HMTtC7B0UgZWi2JadiLbKltxevw8sGo/C8elsCAvhbUlXevUhNzwyDru+vd2atvd2KyKlJjInif4PLDlCRi/FKKTj/i5hThVSXA/Dm0uL798fTfz732Hrz3Vc13yQ+vYKTGRpAanunevu4c6TOvaXXxyoIn1pc3cce4EomxWxqaYzH1fXQdVLU5yU2LISLCH6+3TshLCZZnQRKCU2EjuenEHLq+fssZOxqXHkRIbyQVTRgHB4G43wT0rMZrPn5HLvVdMY29dB/e8uivcLq8/wKMfleD0+NFa8+rWahYXpHHrOeOxWhRrwsG9s0dJBgjvYlTZ4gz/Igv9Evr3FlMC6p5JzxiTFB6FODUrAYC5ucm4fQFiIq0syE8BYE5uMjsqW/nmM1to6vTwzWWFLJmUzoH6Tho73Hywt549te20ubys3lPPu0V11La5yIiPOnzzjV0vQUctnHErQoxEEtz78cz6cnZUHj4Er7rVydUPruHvH5QQYVG8u7u2x1IBoaGM07JNsEqOjSQ1uFBW98lHZcHgXt/uZvNBU8u+Yo7ZXzzWbsodK7dXE9AwJiWG9OAviNTYSCZkxIeDZii433L2ODy+AG/vrsXr14xLM9n/ty8o5DsXFJIUYwsvrzsuPRaLRfH5M3I5f0omRTVdmfu7RXXc/count94kE3lLVS2OLlsRhZRNivj02MpqmmjscNNQ4ebgoyeE4NCE6V+8K/tLPzlOzR0uMO/hA42HV6WibNHUJgRT1KMjdHB43PGmmx68YQ07BFmvZdbzx7PvNxk3thZw5yxSSzIT2F+ngn87xXXc/PjG/jhv7az7kATAQ3VrS62VbSS0Vtn6toHIXWCydyFGIEkuPchEND894s7eWJtWY/jWmtufHQDFc1OHrthAT+8ZDKdHj+7uo3Nrms3mfvZBWav2JRYG+nxZgz4muByt+0ub3hSU327m+oWV4/MGuCS6aPZV2dGiEzIiAsHqtFJUeSmxlDT5sLl9VPa2Em8PYJzJ2YA8OLmKgDy001wH5cexx1LC1BKha/ffRPowsw46tvd4TH5oSV5V26v4ZWtVURGWLhgaiYAk0YlsLu6nS3B8tCssUk9vj+jEqOwKJOte/2aXVVtPfoGIiMsJMf0HJ1yy9njuH3J+HB5Z35eMrGRVi6b2bVKY2KMjce/vIDvXFDIvVdMRynF9OxEIq0W7nu9CJc3wIayZp7b2LWf+766jh4lIFor4OnPQ+UGk7Vb5L+AGJlGzE92ZYuTbz2zpcdIlONV3+HG4w/QcshEn08ONLG7uo0fXzqFxQVpLAhmj+tKuka9hDL3s8LB3U6UzcqNi/P516ZKNpU3h4c5QjC4tzrDmWvI3Sumsv6uZTx9y0JmjUkKZ+6jEqLDZZuKZgclDZ3kpcWSkxxNSmwkq4rNboehzL27+GDNfVx613MFwWn5e4J1908OmM+ytqSRl7ZUsnRiRjjjnzQ6nsoWJ6v31BNhUeFt60JsVgsXTx/NzWflh69Z3uRgUnBMe2aCvUeNHuDKuTnccvb48OPUODsb//v8HsEdwB5h5Y6lBUwebf4iirKZ+nxDhzvcAf3mzlrm5SaH+xkyE6JMQX/r0/DAp2D/u3DeT2DejYd9b4QYKUZMcF9VXMe/Nlf2yJ6PV0VwOF+zo+fY9MfXlJIUY2PFLBN4RiVGMTYlpkdwr2t3EWePYG5uMtfMG8OSiSbIf23pBDIT7PzkpZ3hertSZthkVYsrPIO0u9Q4OwvHpaKUIiOYhWYlRTE2GMxKGxyUNprgrpRiRk4ivoAmISoivGZ6d6HMfVxaV+Y+MRTc6zpocXgoqmnjommjCGhodnjDnxVg8igTWP+9uZLJoxOIjjx8mdz//dwc7rpkCmlxkawvbaLV6eWymVlE26w9M+k+RNkGtvzuvODQya+eO4E5wb8izp2UEf4FMFWVwKOXwr+/AplT4baP4axvyfK+YkQbun3WBlltcMjbQCfkdOcPaL7xzBauPzOXubkp4eOhiTjdM/fqVif/2VXLTYvzewSf+XkpvFdcxwOr9hEbGUFdm5uMeDuRERbu/8yM8Hmx9gjuPK+QH/57Oy9tMaWTwoz4cOY+J7dnieNQ4cw9MYpJo+KJjLCwak8dlc1Orphl1kefkZPEquJ68tPjDsuQAbKSorFZFYWjuoL76MQo4u0R7K1tZ12JHa3h+jPz2F3dRn27O1zuAZO5A7S7fMwe23d7CzPjeX+P2eVofHoct54zPrxv6WC5cm4Ode1uLpuZhdPrZ1N5M8sSK8mJ/hC/7WOu2PgRxKTCJb+BuTdIUBenhRET3EMrDYbGNR+NqhYnr2ytIi0ukrm5KbQ6vcRGWsMTcboH9/f31OMPaK6al9PjGgvyk3lhUwW/eqMYe4SFcelxvXfkAZfOHM1PX9nJW7tqSY+3k5saw+6aNpodXkYnHp65d5cevGZWYjQxkREsnpDGCxsrCWjIC5ZgZuaYMsn4XkoyABdPH83ssUlkxHdl0EopJmTGUVzTjs1qwR5hYdbYJO5eMZVWp7dHdj4qIYrEaButTu+Agnto2OTYlBiWTxvV5/lHLeCnMDHA75bFw+4X+ELzBj6T+Q4xL+9jIuCy2KiccjM5K34EUYn9Xk6IkWIEBXd3+Ks/oFm9p45zJ2aglGJPbfthY7G7C2Xoe2rb8fgCnPs/q7jprPzwhJvuZZnSRgc2qwoveBVy3uRMzi6sYVpWAg+s2s/u6jZWzOx9y7aEKBsXTB3FK1uryE+NJSPBzn92mffKSuq7ZDEjO5EFeSnMDw4PXDY5k3eLTH09NC5+5pgkLAomZMb1eg2rRZGTHHPY8YmZ8by2vZri2nYWjkvFHmFlSbeMPUQps4vR2pImZo/pe4x4aO0YIFxGOmrudmgpN7fmUmg6AB4HdNZB2cfg6VqSwGKLISZ7Lpz9dVy5S3h0m4sbzy6AXhYNE2IkGzHBPVSWqW1z8V5RHTc9voEXbvsU9ggrl/7pQ3586RRuXJzf62tDGXpxjVlpsanTw5r9jUQEx0a7fQGcHj/RkVbKGjsZkxxDhLVnsEiLs/P4jQsIBDT/2lRJTZur9/VMgq6ck80rW6vITY0hPa4roPeXuafG2Xn21k+FHy+bnMEP/23u5wcz9bQ4O8/demaPwDoQBZnxtK8/SJw9gp+umNrnuQvyU6hscZLbT8AuDP6CSY2N7DEKCABnC7RVmfHmHXXgagGvA7xOc7x2pwnmzkN2n0FCpPoAAA+qSURBVLInBG9xMONqSBkP0UkweiakTwareZ8o4FYZ6ShOUyMmuIfKMrVtLvYGhw6WNzmwBLP1X79ZzPlTMhmTcngwCmXuDR3ucBa89WBLj4k2LU4P0ZHRlDY4+gxoFovi4umjefijkt6nvActnpDGogmpnDspo8faKFm9BXe/FzobTKbq85iasbsNtCYjNp2zszV7Gn0kB5rBY4Lp3Jh66GwGXyJEJQxoYaw5Y5OwWhS//syMcInnMIEAuFu5c34Mt03LQtVsg/YaE6A9nSaL9nSaW2cDMzsaeMrWgM0aC88/BV4XtFdBU4kJ5kcSk2Y6P6deDkljg7dcc4tNk7XXhejHiAjuLq8/HCBr2lyUNJjgXtXiCscApeDe13bz4HVzD3t99y3hnt9kxki3uXy0ucwY6Zo2F82dXkYlmP0+QzMmj2TFrCwe/qiEMcl2aNwPjfvMraXcZKjKQkSEnSdS/FDso7alk/+zNeDDSvb7r4E9FtqroWGvCZrO5j7f7/HQnd/0cZIt1gT5qEST9UYFg77fYwJxVCKzURRPcxJR9jZURpmM2dFkvjqbzX1XC+gAEfTxw2OJgMhYiEkjIiaVWBukWFqhajNERENcBkz7L0jOh8RsiBsFcZlmGYDIGIiIkuAtxHEaEcE91ImaGG2jttUVnq1Z2eLEoiA5xsayyZms3lPf6+srW5z8//buPTiu6j7g+Pen1eppafWwLevlN0nwA9uKje0gMnk0sWJDSWdocJhmnIQZZkLIkHaSFAYmpbRNA5kEyGPKUEJJSMEESAJDnSaOgRIeQTjE2MaObSEZW5Zl2dbLtiTL1v76xz0rrTZ6s2/9PjM7e+/dK9+fzl39fPacs+dUBHJo6erjaHsv1SW5g9+kXFpRSGt3H529/Zw8e56e/oHB5g/Aq4mea4OzJ91zGytPH2Lfgnpyn9kLF8KWkMsq8BIbOlQDz8gkMAAL5Tw5GUF8jUe8n8mbCbMvhfm1kD9r6OHPheBFL0GDN6vhuZNeU4Y/10vUOgCBaggOeDX8vq7hj/Pd0HMK2t8BX5aXiDuPAJDpy4Ijr3qfFnJLIK/Yew5Uuf0S7zkrH/x53jVnlEFBGWQXeP+JZA4fDVPW3ed1yObYtLrGxEt6JHfXJLOiuoiXDp5kX4s31r2lsxfBG/o3r9SbqTDUdh7uWEcvq+YVc85NH/s3q6p46PeN9PQPsLQywI4/t9HZc4F3T51jiRymtm0X/KrFq4m27QMiJg7zZZNXfhnUfA7mXAYzL/HahfNKRqyRdnT1suHfn2dFVYBnbq6NRREl1FjNU8aY2EiL5B6aWndFVYCXDp7knJve1kvuwtzSvMG29qMdPcMWSQ4GlZbOPjYsncP7ywqoP9xOzdwillcGeLOpjVr/AS74nmPZyz9mVucetmW3wS4gfzbMWQaXXuXVavNnezXrGbO8ZobMiY/lnunGro/XmWqMMROVFsk91CyzompozHVowYgMEdYvKh0cJnjk9PDkfspNMVBZnMtATwcZR/ax5kQTFb2/oSK7nhkv9nG5Hzq7q2kqWMWj5+Zy19e/hj8QvfHafl8G7y8rYHmVjcM2xkRHeiT37j7ys3wsCpt6tnbxTJ5w85pXFg3Nw/Juew+PvNLE/+1v5u7aDPqa3uC7/u3UvXqM/DNNkAU8D4sKqzk872pmrP806x7r4eply2jp6uPt3q6oJvaQbbdcSeSstMYYM1UTSu4iUgfcD/iAh1T12xGvZ+MN2vggcBq4TlUPRzfU0Z3o7qMskDM4Z4kIrF9UOpTcC30UnznI9dmvsPytpwm07+b6i41kbfWab3IyigiWroU1fwcVK2H2EnwF5SwKtY/n7vDa3E+fG/wEEG0+y+zGmCgaN7mLiA/4EfAJoBl4Q0SeVdV9YafdAHSo6mIR2QzcDVwXi4C3vryfB15sYPttm/C7LxK1dnkLLuf6lBU5rSzO7qTm9BG+53+O5dLE4mdaER3gWwK9p3LZNbCAY7M+w4tnq/ltVzWtlLD3s3UQ+SUbpyjPT2t3Hwdbz/L5K2bG4tcyxpiomkjN/XKgQVUbAURkK3ANEJ7crwHudNtPAT8UEdHxFuacgktafsWvL3yPC09ejb+0mmNHG/mXE7up9p+Bb53hGfrhPPAy5GYUsit4CRWXX0t+5XLuqhceacgmSAaPbVzLV2YXsPPheqSn/y+/PRmmOC+L15va6R8Ism7h2GPcjTEmGUwkuVcCR8P2m4G1o52jqhdFpAsoBU6FnyQiNwI3AsydO3dKAQer1/P07ivZ3Lid4MFeGCikP3s+2R+ohYJSTuQuJKNkAaWzq7ji3gOA8OcNdZAhZDbvJ9jQiN8nrKouJjfLxy9u+tDg4tSjKcrz038xSIbA6vmW3I0xyS+uHaqq+iDwIMDq1aunVKufMb+GOy7eQPGmH9DQdo57dxxiz22fJNt9QaYs7NzyIu+LOaH1M0OdqssrA4Nj3XP8vnHnDS9yiysvrQhQaF/EMcakgIkk92NAddh+lTs20jnNIpIJBPA6VqMutFLR8e7zNHf2Mrsge3CFoEjLKocPLQwl9zXjTB8QKbQk3NpJ/pwxxiTKRJL7G8AlIrIAL4lvBq6POOdZYAvwGnAt8Hws2tvBm2Igx59Ba1cfzR29I04EFnL/dSuH7S+tKKSyKJe6pZMbyljkkvu6haWTD9gYYxJg3OTu2tBvBn6DNxTyYVV9W0TuAnaq6rPAj4FHRaQBaMf7DyAmRITyQC7Hu/s42tHDB+eNPp945LS8pTOyeeXWyc8Bu2puMUvKC1lrnanGmBQxoTZ3Vd0GbIs49s2w7T7gb6Mb2ujmFObQ3NHL8a4+qkdYdCLa1swvYdstV8b8OsYYEy0puTxNeSCH/S3dDASVqmKbj8UYYyKlZHKfE8ihfyAIMGabuzHGTFcpmdxDI2YAq7kbY8wIUjK5z3FT44rYNLnGGDOSlEzuoZp7eWEOWbaqvTHG/IWUzIxzXHKvisNIGWOMSUUpmdxL8rLI8mVQVWJNMsYYM5KUXKwjI0O4fdOltnKRMcaMIiWTO8CWD81PdAjGGJO0UrJZxhhjzNgsuRtjTBqy5G6MMWnIkrsxxqQhS+7GGJOGLLkbY0wasuRujDFpyJK7McakIYnRUqfjX1jkJPDuFH98JnAqiuFEU7LGZnFNjsU1eckaW7rFNU9VZ413UsKS+3shIjtVdXWi4xhJssZmcU2OxTV5yRrbdI3LmmWMMSYNWXI3xpg0lKrJ/cFEBzCGZI3N4poci2vykjW2aRlXSra5G2OMGVuq1tyNMcaMwZK7McakoZRL7iJSJyIHRKRBRG5NwPUPi8geEdklIjvdsRIR2S4ih9xzsTsuIvJ9F+tuEamJYhwPi0ibiOwNOzbpOERkizv/kIhsiWFsd4rIMVduu0RkY9hrt7nYDojIhrDjUbvXIlItIi+IyD4ReVtEbnHHE15mY8SW6DLLEZF6EXnLxfXP7vgCEXndXeMJEclyx7PdfoN7ff548UY5rkdEpCmsvFa64/F+//tE5E8i8pzbT0x5qWrKPAAf8A6wEMgC3gKWxDmGw8DMiGP3ALe67VuBu932RuDXgADrgNejGMeHgRpg71TjAEqARvdc7LaLYxTbncDXRjh3ibuP2cACd3990b7XQDlQ47YLgIPu2gkvszFiS3SZCTDDbfuB111Z/BzY7I4/AHzJbd8EPOC2NwNPjBVvDOJ6BLh2hPPj/f7/B+Ax4Dm3n5DySrWa++VAg6o2qmo/sBW4JsExgRfDT9z2T4BPhx3/qXr+ABSJSHk0LqiqLwHt7zGODcB2VW1X1Q5gO1AXo9hGcw2wVVXPq2oT0IB3n6N6r1X1uKq+6bbPAPuBSpKgzMaIbTTxKjNV1bNu1+8eCnwMeModjyyzUFk+BXxcRGSMeKMd12jidi9FpArYBDzk9oUElVeqJfdK4GjYfjNj/xHEggK/FZE/isiN7liZqh53261AmduOd7yTjSPe8d3sPhY/HGr+SERs7uPvKrwaX1KVWURskOAyc00Mu4A2vOT3DtCpqhdHuMbg9d3rXUBpPOJS1VB5/Zsrr3tFJDsyrojrx+Je3gd8Awi6/VISVF6pltyTQa2q1gCfAr4sIh8Of1G9z1UJH1+aLHGE+Q9gEbASOA58NxFBiMgM4Gngq6raHf5aostshNgSXmaqOqCqK4EqvNrjB+Idw0gi4xKRZcBtePGtwWtq+cd4xiQiVwFtqvrHeF53NKmW3I8B1WH7Ve5Y3KjqMffcBvwS7w1/ItTc4p7b3OnxjneyccQtPlU94f4gg8B/MvQxM26xiYgfL3n+t6r+wh1OijIbKbZkKLMQVe0EXgDW4zVrZI5wjcHru9cDwOk4xVXnmrdUVc8D/0X8y+sK4K9F5DBek9jHgPtJVHlNpcMgUQ8gE6/TYwFDHUZL43j9fKAgbPtVvDa67zC8U+4et72J4R059VGOZz7DOy0nFQde7aYJrzOp2G2XxCi28rDtv8drUwRYyvDOo0a8jsGo3mv3u/8UuC/ieMLLbIzYEl1ms4Ait50L/B64CniS4R2EN7ntLzO8g/DnY8Ubg7jKw8rzPuDbCXz/f4ShDtWElFfUEk28Hng93wfx2v5uj/O1F7pCfwt4O3R9vHayHcAh4HehN4h7M/3IxboHWB3FWB7H+6h+Aa9N7oapxAF8Ea/DpgH4Qgxje9RdezfwLMMT1+0utgPAp2Jxr4FavCaX3cAu99iYDGU2RmyJLrPLgD+56+8Fvhn2d1Dvfv8ngWx3PMftN7jXF44Xb5Tjet6V117gZwyNqInr+9/9ux9hKLknpLxs+gFjjElDqdbmbowxZgIsuRtjTBqy5G6MMWnIkrsxxqQhS+7GGJOGLLmbaU1EvioieYmOw5hos6GQZlpz3yZcraqnEh2LMdFkNXczbYhIvoj8j5sHfK+I/BNQAbwgIi+4cz4pIq+JyJsi8qSb7yU0j/894s3lXy8iixP5uxgzHkvuZjqpA1pUdYWqLsP7inoL8FFV/aiIzATuAP5KvcnhduLNzR3SparLgR+6nzUmaVlyN9PJHuATInK3iFypql0Rr6/DWyjhFTed7BZgXtjrj4c9r495tMa8B5njn2JMelDVg26JtY3Av4rIjohTBG9u8M+O9k+Msm1M0rGau5k2RKQC6FHVn+HNBlkDnMFb2g7gD8AVofZ010b/vrB/4rqw59fiE7UxU2M1dzOdLAe+IyJBvBkrv4TXvPK/ItLi2t0/DzwetorPHXizLAIUi8hu4DwwWu3emKRgQyGNmQAbMmlSjTXLGGNMGrKauzHGpCGruRtjTBqy5G6MMWnIkrsxxqQhS+7GGJOGLLkbY0wa+n+C9Yh+W7HIEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ##### Training #####\n",
    "last_saved_step = saved_global_step\n",
    "num_steps = saved_global_step + train_steps\n",
    "losses = []\n",
    "dev_losses = []\n",
    "steps = []\n",
    "pearsons = []\n",
    "dev_pearsons = []\n",
    "\n",
    "print(\"Start training ...\")\n",
    "try:\n",
    "    for step in range(saved_global_step + 1, num_steps):\n",
    "        start_time = time.time()\n",
    "\n",
    "        indexes = np.random.choice(n_data, batch_size)\n",
    "        batch_source = train_data[indexes]\n",
    "        batch_labels = train_labels[indexes]\n",
    "\n",
    "        feed_dict = {\n",
    "            source_ids: batch_source,\n",
    "            targets: batch_labels,\n",
    "        }\n",
    "\n",
    "        loss_value, _ = sess.run([loss, optim], feed_dict=feed_dict)\n",
    "        dev_loss = sess.run(\n",
    "            loss, feed_dict={source_ids: dev_data, targets: dev_labels})\n",
    "        losses.append(loss_value)\n",
    "        dev_losses.append(dev_loss)\n",
    "\n",
    "        duration = time.time() - start_time\n",
    "        if step % print_every == 0:\n",
    "            train_mean_pearson = eval_mean_pearson(\n",
    "                source_ids, predictions, sess, batch_source, batch_labels)\n",
    "\n",
    "            # evaluation on dev data\n",
    "            dev_mean_pearson = eval_mean_pearson(\n",
    "                source_ids, predictions, sess, dev_data, dev_labels)\n",
    "\n",
    "            steps.append(step)\n",
    "            pearsons.append(train_mean_pearson)\n",
    "            dev_pearsons.append(dev_mean_pearson)\n",
    "\n",
    "            info = 'step {:d} - loss = {:.5f}, dev_loss = {:.5f}, '\n",
    "            info += 'r = {:.4f}, dev_r = {:.4f}, ({:.3f} sec/step)'\n",
    "            print(info.format(step, loss_value, dev_loss,\n",
    "                  train_mean_pearson, dev_mean_pearson, duration))\n",
    "\n",
    "        if step % checkpoint_every == 0:\n",
    "            save(saver, sess, logdir, step)\n",
    "            last_saved_step = step\n",
    "\n",
    "except KeyboardInterrupt:\n",
    "    # Introduce a line break after ^C so save message is on its own line.\n",
    "    print()\n",
    "\n",
    "finally:\n",
    "    if step > last_saved_step:\n",
    "        save(saver, sess, logdir, step)\n",
    "\n",
    "    # plot loss\n",
    "    plt.figure()\n",
    "    plt.plot(losses, label=\"train\")\n",
    "    plt.plot(dev_losses, label=\"dev\")\n",
    "    plt.title(\"squared loss\")\n",
    "    plt.xlabel(\"step\")\n",
    "    plt.legend()\n",
    "    plt.savefig(loss_fig)\n",
    "\n",
    "    # plot pearson's r\n",
    "    plt.figure()\n",
    "    plt.plot(steps, pearsons, label=\"train\")\n",
    "    plt.plot(steps, dev_pearsons, label=\"dev\")\n",
    "    plt.title(\"pearson's r\")\n",
    "    plt.xlabel(\"step\")\n",
    "    plt.legend()\n",
    "    plt.savefig(pearson_fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "# Load Data\n",
    "print(\"Loading data ...\")\n",
    "infer_filename = config[\"inference\"][\"infer_source_file\"]\n",
    "max_length = config[\"inference\"][\"infer_source_max_length\"]\n",
    "infer_data, _ = loadfile(infer_filename, max_length)\n",
    "word_ids = pd.read_csv(\"emodata_word_ids\", header=None, sep=\"\\t\")\n",
    "\n",
    "# id_0 represents all-zero token\n",
    "embed_shift = 1\n",
    "infer_data += embed_shift\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start inferring ...\n",
      "\tDone.\n"
     ]
    }
   ],
   "source": [
    "print(\"Start inferring ...\")\n",
    "emo_predicts = []\n",
    "infer_batch_size = config[\"inference\"][\"infer_batch_size\"]\n",
    "n_data = infer_data.shape[0]\n",
    "n_pad = n_data % infer_batch_size\n",
    "if n_pad > 0:\n",
    "    n_pad = infer_batch_size - n_pad\n",
    "\n",
    "pad = np.zeros((n_pad, infer_data.shape[1]), dtype=np.int32)\n",
    "infer_data = np.concatenate((infer_data, pad))\n",
    "\n",
    "for ith in range(int(len(infer_data) / infer_batch_size)):\n",
    "    start = ith * infer_batch_size\n",
    "    end = (ith + 1) * infer_batch_size\n",
    "    batch = infer_data[start:end]\n",
    "\n",
    "    preds = sess.run(predictions, feed_dict={source_ids: batch})\n",
    "    emo_predicts.append(preds)\n",
    "\n",
    "emo_predicts = np.concatenate(emo_predicts)[:n_data]\n",
    "np.savetxt(config[\"inference\"][\"output_path\"], emo_predicts, fmt='%.5f')\n",
    "print(\"\\tDone.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BiLSTM + SelfAttn, RanInit, Pretrained on Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
